src/dctv/block_builder.h

// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "dctv.h"

#include <functional>
#include <stddef.h>

#include "npy.h"
#include "vector.h"

namespace dctv {

struct Block;
struct QueryCache;

// Accumulates data for storing into blocks.
struct BlockBuilder final {
  // Accumulate something into the output.  DELIVER_BLOCK is a
  // function called with one argument, an unique_obj_pyref to a new
  // block, every time we produce a block.  IS_EOF indicates whether
  // we're done with the stream.
  //
  // DATA describes the block data.
  // TODO(dancol): describe acceptable data formats
  using OutputFunction = std::function<void(unique_obj_pyref<Block> block)>;

  // Make a new block builder.  DTYPE is a dtype.  QC is the query
  // cache we use for making output array hunks; the caller is
  // responsible for ensuring that QC remains valid for all calls into
  // BlockBuilder except destruction.
  BlockBuilder(QueryCache* qc,
               unique_dtype dtype,
               OutputFunction output_function);

  // These functions may call into the pre-configured output function!
  void add(pyref data, bool is_eof);
  void set_block_size(npy_intp block_size, npy_intp maximum_block_size);

  inline bool is_eof() const noexcept;

  // Return the offset of the Python field for the dtype, useful for
  // member setup.
  static constexpr size_t get_dtype_offset();
  inline const unique_dtype& get_dtype() const noexcept;
  inline npy_intp get_block_size() const noexcept;
  inline npy_intp get_partial_size() const noexcept;

  int py_traverse(visitproc visit, void* arg) const noexcept;

 private:

  void add_1(pyref data, bool is_eof);
  void flush_partial_block();
  unique_pyarray make_block_array(unique_dtype dtype, npy_intp size);

  OutputFunction deliver_block;
  QueryCache* qc;
  unique_dtype dtype;
  unique_pyarray partial_data;
  unique_pyarray partial_mask;
  npy_intp partial_size = 0;
  npy_intp block_size;
  npy_intp maximum_block_size;
};

inline void init_block_builder(pyref m) {}

}  // namespace dctv

#include "block_builder-inl.h"