Assignments, Actual

Milestone 5

For this week, your goal is to complete any missing bits in your implementation and write a distributed application that we call "7 degrees of Linus". The goal of that application is to compute all the people who worked on projects up to seven degrees of Linus Torvalds. Degree 1, is people who worked on the same projects as Linus, Degree 2 is people who worked on projects with Degree 1 folks. And so on.

This application will be discussed in a course video.

The deliverables are updated project document and code bases.

Milestone 4

For this week you should complete the distributed key value store implementation including the network layer. The goal will be to support a simple distributed word count application. The application was written by the company sales rep, as they were explaining eau2 to the client. It uses a number of imaginary APIs that you may not have. Feel free to ignore anything that clashes with your design. If this code is not helpful, you can reimplement something that works better for you. The code should be mostly correct, but since it is neither thoroughly documented, nor exhaustively tested, you should beware.

The input of the wordcount application is a single very large text file containing words separated by spaces. The output should be the list of different words and their counts. (The code provided only prints the number of different words)

This application will be presented in a course video.

class FileReader : public Writer {

public:

/** Reads next word and stores it in the row. Actually read the word.

While reading the word, we may have to re-fill the buffer  */

void visit(Row & r) override {

ssert(i_ < end_);

assert(! isspace(buf_[i_]));

size_t wStart = i_;

while (true) {

if (i_ == end_) {

if (feof(file_)) { ++i_;  break; }

i_ = wStart;

wStart = 0;

fillBuffer_();

}

if (isspace(buf_[i_]))  break;

++i_;

}

buf_[i_] = 0;

String word(buf_ + wStart, i_ - wStart);

r.set(0, word);

++i_;

skipWhitespace_();

}

/** Returns true when there are no more words to read.  There is nothing

more to read if we are at the end of the buffer and the file has

all been read.     */

bool done() override { return (i_ >= end_) && feof(file_);  }

/** Creates the reader and opens the file for reading.  */

FileReader() {

file_ = fopen(arg.file, "r");

if (file_ == nullptr) FATAL_ERROR("Cannot open file " << arg.file);

buf_ = new char[BUFSIZE + 1]; //  null terminator

fillBuffer_();

skipWhitespace_();

}

static const size_t BUFSIZE = 1024;

/** Reads more data from the file. */

void fillBuffer_() {

size_t start = 0;

// compact unprocessed stream

if (i_ != end_) {

start = end_ - i_;

memcpy(buf_, buf_ + i_, start);

}

// read more contents

end_ = start + fread(buf_+start, sizeof(char), BUFSIZE - start, file_);

i_ = start;

}

/** Skips spaces.  Note that this may need to fill the buffer if the

last character of the buffer is space itself.  */

void skipWhitespace_() {

while (true) {

if (i_ == end_) {

if (feof(file_)) return;

fillBuffer_();

}

// if the current character is not whitespace, we are done

if (!isspace(buf_[i_]))

return;

// otherwise skip it

++i_;

}

}

char * buf_;

size_t end_ = 0;

size_t i_ = 0;

FILE * file_;

};

/****************************************************************************/

class Adder : public Reader {

public:

SIMap& map_;  // String to Num map;  Num holds an int

Adder(SIMap& map) : map_(map)  {}

bool visit(Row& r) override {

String* word = r.get_string(0);

assert(word != nullptr);

Num* num = map_.contains(*word) ? map_.get(*word) : new Num();

assert(num != nullptr);

num->v++;

map_.set(*word, num);

return false;

}

};

/***************************************************************************/

class Summer : public Writer {

public:

SIMap& map_;

size_t i = 0;

size_t j = 0;

size_t seen = 0;

Summer(SIMap& map) : map_(map) {}

void next() {

if (i == map_.capacity_ ) return;

if ( j < map_.items_[i].keys_.size() ) {

j++;

++seen;

} else {

++i;

j = 0;

while( i < map_.capacity_ && map_.items_[i].keys_.size() == 0 )  i++;

if (k()) ++seen;

}

}

String* k() {

if (i==map_.capacity_ || j == map_.items_[i].keys_.size()) return nullptr;

return (String*) (map_.items_[i].keys_.get_(j));

}

size_t v() {

if (i == map_.capacity_ || j == map_.items_[i].keys_.size()) {

assert(false); return 0;

}

return ((Num*)(map_.items_[i].vals_.get_(j)))->v;

}

void visit(Row& r) {

if (!k()) next();

String & key = *k();

size_t value = v();

r.set(0, key);

r.set(1, (int) value);

next();

}

bool done() {return seen == map_.size(); }

};

/****************************************************************************

* Calculate a word count for given file:

*   1) read the data (single node)

*   2) produce word counts per homed chunks, in parallel

*   3) combine the results

**********************************************************author: pmaj ****/

class WordCount: public Application {

public:

static const size_t BUFSIZE = 1024;

Key in;

KeyBuff kbuf;

SIMap all;

WordCount(size_t idx, NetworkIfc & net):

Application(idx, net), in("data"), kbuf(new Key("wc-map-",0)) { }

/** The master nodes reads the input, then all of the nodes count. */

void run_() override {

if (index == 0) {

FileReader fr;

delete DataFrame::fromVisitor(&in, &kv, "S", fr);

}

local_count();

reduce();

}

/** Returns a key for given node.  These keys are homed on master node

*  which then joins them one by one. */

Key* mk_key(size_t idx) {

Key * k = kbuf.c(idx).get();

LOG("Created key " << k->c_str());

return k;

}

/** Compute word counts on the local node and build a data frame. */

void local_count() {

DataFrame* words = (kv.waitAndGet(in));

p("Node ").p(index).pln(": starting local count...");

SIMap map;

Adder add(map);

words->local_map(add);

delete words;

Summer cnt(map);

delete DataFrame::fromVisitor(mk_key(index), &kv, "SI", cnt);

}

/** Merge the data frames of all nodes */

void reduce() {

if (index != 0) return;

pln("Node 0: reducing counts...");

SIMap map;

Key* own = mk_key(0);

merge(kv.get(*own), map);

for (size_t i = 1; i < arg.num_nodes; ++i) { // merge other nodes

Key* ok = mk_key(i);

merge(kv.waitAndGet(*ok), map);

delete ok;

}

p("Different words: ").pln(map.size());

delete own;

}

void merge(DataFrame* df, SIMap& m) {

Adder add(m);

df->map(add);

delete df;

}

}; // WordcountDemo

Deliverable: an updated report and a snapshot of the repository that runs this application.

Milestone 3

For this week your focus could be on distributing the key value store. That means eau2 should be able to run with multiple KV stores, and thus multiple instances of the application. While you can do this by plugging a network layer, you could also break down the work and "fake" the network and run multiple KV stores in the same process in different threads. The advantage of the pseudo-network approach is that debugging is a bit easier and you can focus on the design of the rest of the system without worrying about networking issues.

The goal for this week is to support the execution of the example shared in Milestone 1.

Deliverable: an updated report and a snapshot of the repository.

Milestone 2

For this week, we encourage you to focus on the key value store in the context of a single-node system (this means still no networking or concurrency required, if you are done early, you can start on those).

We envision two possible APIs to the store: the client interface which works on data frames only and (optinally) an internal, or private, interface that is used to store data hidden from the client.

A client program needs to be able to get, put, and wait_and_get on the KV store. Each of these operations take keys and (return) dataframes. Internally, you may want to store chunks of data frames or columns, etc. How you design this interface is up to you, as long as you support the client operations mentioned above.

A target for this week could be to get code similar to the following snippet to run on your system:

class Trivial : public Application {

public:

Trivial(size_t idx) : Application(idx) { }

void run_() {

size_t SZ = 1000*1000;

double* vals = new double[SZ];

double sum = 0;

for (size_t i = 0; i < SZ; ++i) sum += vals[i] = i;

Key key("triv",0);

DataFrame* df = DataFrame::fromArray(&key, &kv, SZ, vals);

assert(df->get_double(0,1) == 1);

DataFrame* df2 = kv.get(key);

for (size_t i = 0; i < SZ; ++i) sum -= df2->get_double(0,i);

assert(sum==0);

delete df; delete df2;

}

};

Deliverable: A snapshot of your repository containing an updated version of your report and the current code base. All of the expectation from last week still hold. Grading on the report will look at the details of the description of the classes you have implemented. Grading for the code will be based on your ability to make the above example run, your tests and clean valgrind results. As previously bound the report writing time to 3 hours,

Milestone 1

Your goal is to design the architecture of the eau2 system from the requirements elicited by the company’s CEO when he sold the, yet-to-be-built, technology to their first customer. What follows is all that engineering has to go on, it has been made clear that losing this customer would likely result in an untimely end for the company.

Requirements call.

First design meeting notes.

class Demo : public Application {

public:

Key main("main",0);

Key verify("verif",0);

Key check("ck",0);

Demo(size_t idx): Application(idx) {}

void run_() override {

switch(this_node()) {

case 0:   producer();     break;

case 1:   counter();      break;

case 2:   summarizer();

}

}

void producer() {

size_t SZ = 100*1000;

double* vals = new double[SZ];

double sum = 0;

for (size_t i = 0; i < SZ; ++i) sum += vals[i] = i;

DataFrame::fromArray(&main, &kv, SZ, vals);

DataFrame::fromScalar(&check, &kv, sum);

}

void counter() {

DataFrame* v = kv.waitAndGet(main);

size_t sum = 0;

for (size_t i = 0; i < 100*1000; ++i) sum += v->get_double(0,i);

p("The sum is  ").pln(sum);

DataFrame::fromScalar(&verify, &kv, sum);

}

void summarizer() {

DataFrame* result = kv.waitAndGet(verify);

DataFrame* expected = kv.waitAndGet(check);

pln(expected->get_double(0,0)==result->get_double(0,0) ? "SUCCESS":"FAILURE");

}

};

You are to write a document describing the architecture of the system as you envision it. The section of that document are:

• Introduction: where we give a high-level description of the eau2 system.

• Architecture: where we describe the various part of eau2 at a high-level.

• Implementation: where we describe how the system is built, this can include a description of the classes and their API, but only the class you deem relevant and worth describing. (For example: do not describe the utility classes.)

• Use cases: examples of uses of the system. This could be in the form of code like the one above. It is okay to leave this section mostly empty if there is nothing to say. Maybe just an example of creating a dataframe would be enough.

• Open questions: where you list things that you are not sure of and would like the answer to.

• Status: where you describe what has been done and give an estimate of the work that remains.

Be precise and compact in your writing, avoid filler text. If you are not sure what should go in a section write less. You will update this document each week, so it will gradually be fleshed out. Best if you use markdown. Allocate no more than three hours to the writing. What you can do in that time is what you should deliver. If not perfect, it’ll improve next week.

The document is as much for you as it is for us. Grading will be based on superficial characteristics: spelling, grammar, adherence to the above description. Easy points. The following weeks will pay more attention to content.

Technical debt.

For the implementation you should focus on managing technical debt from your past efforts. You should create a new private GitHub repository for the project and move whichever code you want to retain from past work in that repository. This is a time to clean up your code and write tests. It is fine to simplify the code and only add what you are sure to need. You can always bring in more later.

You should reuse your array, dataframe, adapter classes. Feel free to strip them down to their essence – e.g. you can drop things like col/row names; these can be added later – anything you are not using in your examples or do not see a need for, drop. The smaller the code base the better.

Your project must have a Makefile with targets that:

• build your code

• run test case, there should be a test case that reads data, builds a dataframe, and does some trivial operation

• run valgrind – if you are on a Mac this has to be done within docker.

Deliverable: a snapshot of your repository that contains a directory with your report and a directory with your code.

Grading will be based on the presence of the above mentioned and code and test quality.

Assignment 6

Part 1. Networking, really

For this assignment you have to complete the design and implementation of a network communication layer for a distributed application, informed by your prototype from A5. The system consists of a rendezvous server (registrar) and an arbitrary number of nodes. A sketch of the architecture is provided below:

To participate in the system, a node needs to register with the rendezvous server, providing its IP address (and, optionally, a port) on which it will wait for connections from other nodes. The server will send a list of currently active nodes (a directory) to every active node. Once registered, a node can connect to any other node in the system directly to communicate via a predefined set of messages.

The communication layer for this system needs to support at least the following features:

• Startup, meaning:

  • Rendezvous server startup

  • Node startup and registration

  • Directory update from the server

• Sending messages of arbitrary length and content between nodes

• Teardown - orderly shutdown of the whole system

There are two levels to this task:

• Design of a simple communication protocol. This includes the description of request and response types, as well as their sequencing.

• Design and implementation of an API based on the communication protocol, using sockets for connection management.

The deliverables are: a memo documenting your design and implementation, and an implementation of the communication layer. Tests are for your confidence in your own work.

Submission structure:

Your MEMO.pdf should be written with sufficient details for members of other teams to use your communication layer code. Add some use cases. Pay attention to details.

The API shouldn’t assume any particular IP addresses and ports used for the rendezvous server or the nodes, but you can assume that these will be provided via command line or a config file. Your example drivers can use hardcoded addresses (preferably in a way that’s easy to change) or accept them as command line options.

The API should be complete in the sense that it abstracts and hides the details of establishing a connection, as well as sending, receiving, and validation of requests and responses.

Hints: For documenting the sequencing of communication, you might find sequence diagrams to be useful: example and another example.

Part 2. Serially, really

Your task is to implement a serialization support class and demonstrate how to use it to serialize elements of a DataFrame. The term serialization refers to the transformation of a set of objects in memory into a sequence of bytes (chars) that can be sent through the network or stored on disk. This includes deserialization, that is, the transformation of a sequence of bytes back into an object of the correct type. Your design will likely be used in the near future. The deliverable is a MEMO.pdf describing your design, along with use cases. Code is supporting evidence, in the form of a prototype exemplifying serialization and deserialization. You should be able to serialize at least the following classes (the names can be changed to match your code base):

Submission structure:

You MEMO.pdf should be written with sufficient details for members of other teams to use your serialization API. Pay attention to details.

Assignment 5

Part 1. Concurrency, really

Your goal for Part I is to improve your implementation of pmap() and write a MEMO.pdf describing your experimental evaluation. For this part, we will grade you largely on the report (40 out of 50 points); your code serves as supporting evidence. Your report should be single spaced, well formatted, carefully worded. Approximately 5 pages should suffice. In particular, your evaluation should demonstrate that for two tasks of your choosing, pmap performs as well or better than map, in terms of end-to-end performance. To that end, you will do the following:

• Find or create datafile.txt. You need to have some data to work on. These data can come from some real world site or can be synthetic. To give reliable results the data should be "large enough": a data file of 100 MB in size with at least 10 columns.

• Pick tasks. You need to define two Rower subclasses that can be given to pmap(). The second Rower subclass (this is to say, the "function" this subclass describes) should be a far bit more expensive than the first one.

• Design an experiment to measure the performance increase of pmap over map—using the performance on map as a baseline—on executing these two Rower subclasses. Your experiment should vary the size of the input you use, gradually increasing the number of rows from the data file that you consume. The source code you execute for your experiment should be in {bench.cpp}. You may find it useful for your Makefile to do the timing measurements.

• Run your experiment on two computers with different characteristics. Ideally, one of these would be a laptop and the other a desktop. But if your pair does not have access to a desktop, two differently configured laptops will do.

• Write MEMO.pdf, a report describing the preceding and the results.

Your report should contain the following sections:

• Introduction - describes the goal of the report (a description of an experimental evaluation).

• Implementation - a section describing your implementation of pmap().

• Description of the analysis performed - a section describing your data—why and how you created them, what you are going to measure and how (e.g. repeated numbers of runs, number and size of iterations), and describe also the HW/SW infrastructure you use in your experiment.

• Comparison of experimental results Provide and explain your results in both text and graphical form. Describe the sources of non-determinism.

• Threats to validity What could make your results wrong for someone else?

• Conclusions A section summarizing the work.

Your datafile.txt will be too large to submit via handins. Instead, you should host this file somewhere (e.g. github.com). Your Makefile should pull down this file down as a part of executing it. HINT: This 100MB file will be too large to check into many VCSs. Compress (e.g. zip or tarball) datafile.txt first, and commit that.

Part 2. Networking, prototyped

Eventually, for the project, you will need to have a working network layer. In Part 2 you will prototype a networking layer for inclusion in the upcoming project. "Prototyping" means creating something that works well enough to demo, but likely not a finished product. More importantly, writing up a report describing what you have implemented. This is an open-ended assignment. You will not be graded on how much functionality you implement but rather on the quality of the report describing this implementation.

In term of functionality, you should at least support registration and simple directed communication. For registration, we assume one single server with a known address is contacted by a fixed number of clients. The clients share their address with the server, which then sends all addresses to its clients. Directed communication means that each client can send messages directly to any other client. Your network layer can be written using the POSIX networking library functions, but should be encapsulated in its own class. Server and client need to take IP address as arguments on the command line:

The MEMO.pdf should be a PDF file, single-spaced, no more than two pages long. It should have an Introduction, describing what you have implemented in high-level terms, a Design section, explaning you design and showing your API and a Challenges and Open issues section, telling the readers about problems you faced, and issues that remain to be solved. Write clearly. Assume this is read by your coop boss.

Erata: The deliverables for Part 2 originally listed a single .cpp file, main.cpp. This has been updated with separate source files for the client and server code.

Assignment 4

Part 1. Implementation

You just submitted an API for the dataframe class. However, management chose the instructors’ API. You shall find our API at the bottom of this assignment. Your task is to implement our API in CwC and write tests for your implementation. This is code that you are likely to reuse later.

You are free to add additional classes and methods (of course, you should document them in your README.md, explaining what they are and why you wanted them.)

If you see a mistake in our API, report it in a public Piazza post and suggest the fix for it. The moral-equivalent of PRs are preferred over the moral-equivalent of issues. Each "separate" bug report should be in a separate Piazza post (if a single change in functionality requires edits to multiple lines of code, requesting all of those edits should be in the same post). The class will synchronize on the implementation of this one API, so we will accept few changes, and if we accept one, it will be universal across the course and mandatory. Consequently, we will only entertain bug reports "early on." Absent this, you shall follow our API to the letter.

The following program we provide you in basic_example.cpp should compile and run without change (meaning that dataframe.h must include everything this program needs):

#include "dataframe.h"

int main() {

Schema s("II");

DataFrame df(s);

Row r(df.get_schema());

for(int i = 0; i < 100*1000*1000; i++) {

r.set(0, i);

r.set(1, i+1);

df.add_row(r);

}

assert(df.get_int(0,1) == 1);

return 0;

};

For this assignment, be very careful to use exactly the names as given down to the capitalization and spelling; we will automate the testing of your assignments. Use the helper.h, object.h and string.h classes we provide you. Do not change any of those three files; in the course of testing your submission we will copy our original versions into the top-level of your submission. For similar reasons, your submission should also include exactly the Dockerfile that we give you. Some previous projects had trouble scaling, in particular due to doubling of array sizes when they need to grow. In your implementation, make sure that you have arrays with O(1) element access time, but also which never copy the payload.

The basic_example.cpp we provide you is for some basic sanity checking of your implementation against ours. We do not have an exhaustive test suite. Luckily, you will help us with that. For this assignment you shall use Google Tests for your own private test suites, and you should be thorough. However, in addition to that usual, regular, rigorous testing that you perforce implement, your submission shall include 7 extra Google Test files, with a single test in each file. Each of the 7 tests you write and submit will exercise some part(s) of your implementation of our API. These 7 tests you submit to us should not test or measure the behavior of functionality unspecified by the API. Thoroughly comment these tests! Remember, tests are code too.

It is our intention to take these 7 test files from your submission, and run these tests against every pair’s submitted implementation. We will merge the tests we have written together with the tests from every pair’s submission; this will form our automated test suite. We take our implementation of the API as canonical; we consider tests that do not succeed against our implementation to be "incorrect", and we will throw them out. Of those 7 tests you submit, you can earn an extra point for each submission that you break (excluding your own) with a valid test. (Up to a maximum of 20 points).

Erata: The print method on dataframes was missing. The IntColumn had a field. The default constructor for StringColumn was missing. The specification of the visit method was missing a description of the first argument. Access to out of bound indicies was set to undefined. Duplicate column and row names was set to undefined. The second constuctor of DataFrame was modified to say that columns are created empty. The Row set method’s comment was change to state the strings are external. Some typos.

Your README.md should give:

• an introduction to your implementation,

• an overview of the design choices that you made,

• use cases (the "examples" step from the design recipe) of things one can do with your data frame.

You shall submit Part I in a directory named part1; we detail below the structure of the entire submission.

Part 2. Concurrency

You will submit Part II in a directory named part2.

Copy your completed dataframe.h, to a new file modified_dataframe.h, and in modified_dataframe.h add the following method to your dataframe class:

/** This method clones the Rower and executes the map in parallel. Join is

* used at the end to merge the results. */

void pmap(Rower& r)

Then, in a parallel_map_examples.cpp file, write 2-3 examples which clearly demonstrate that on a multi-core machine your implementation of pmap is asymptotically faster than your implementation of plain map.

The PARALLELMAPREADME.md in your submission for Part II shall:

• have example use-cases of your pmap,

• describe your design for your parallel map,

• describe how the examples in your parallel_map_examples.cpp file demonstrate that pmap is faster, and

• describe issues that you encountered in implementing your parallel map.

You are free to add additional classes and methods (of course, you should document them in your PARALLELMAPREADME.md, explaining what they are and why you wanted them.)

Your submission.zip should have the following directory structure:

We will initially test your assignment by running unzip submission.zip; make. We will of course do more to run your submitted tests against others’ assignments.

Below is our API that you should implement. Note that our API does not deal with missing values. That’s for future work.

/**************************************************************************

* Column ::

* Represents one column of a data frame which holds values of a single type.

* This abstract class defines methods overriden in subclasses. There is

* one subclass per element type. Columns are mutable, equality is pointer

* equality. */

class Column : public Object {

public:

/** Type converters: Return same column under its actual type, or

*  nullptr if of the wrong type.  */

virtual IntColumn* as_int()

virtual BoolColumn*  as_bool()

virtual FloatColumn* as_float()

virtual StringColumn* as_string()

/** Type appropriate push_back methods. Calling the wrong method is

* undefined behavior. **/

virtual void push_back(int val)

virtual void push_back(bool val)

virtual void push_back(float val)

virtual void push_back(String* val)

/** Returns the number of elements in the column. */

virtual size_t size()

/** Return the type of this column as a char: 'S', 'B', 'I' and 'F'. */

char get_type()

};

/*************************************************************************

* IntColumn::

* Holds int values.

*/

class IntColumn : public Column {

public:

IntColumn()

IntColumn(int n, ...)

int get(size_t idx)

IntColumn* as_int()

/** Set value at idx. An out of bound idx is undefined.  */

void set(size_t idx, int val)

size_t size()

};

// Other primitive column classes similar...

/*************************************************************************

* StringColumn::

* Holds string pointers. The strings are external.  Nullptr is a valid

* value.

*/

class StringColumn : public Column {

public:

StringColumn()

StringColumn(int n, ...)

StringColumn* as_string()

/** Returns the string at idx; undefined on invalid idx.*/

String* get(size_t idx)

/** Out of bound idx is undefined. */

void set(size_t idx, String* val)

size_t size()

};

/*************************************************************************

* Schema::

* A schema is a description of the contents of a data frame, the schema

* knows the number of columns and number of rows, the type of each column,

* optionally columns and rows can be named by strings.

* The valid types are represented by the chars 'S', 'B', 'I' and 'F'.

*/

class Schema : public Object {

public:

/** Copying constructor */

Schema(Schema& from)

/** Create an empty schema **/

Schema()

/** Create a schema from a string of types. A string that contains

* characters other than those identifying the four type results in

* undefined behavior. The argument is external, a nullptr argument is

* undefined. **/

Schema(const char* types)

/** Add a column of the given type and name (can be nullptr), name

* is external. Names are expectd to be unique, duplicates result

* in undefined behavior. */

void add_column(char typ, String* name)

/** Add a row with a name (possibly nullptr), name is external.  Names are

*  expectd to be unique, duplicates result in undefined behavior. */

void add_row(String* name)

/** Return name of row at idx; nullptr indicates no name. An idx >= width

* is undefined. */

String* row_name(size_t idx)

/** Return name of column at idx; nullptr indicates no name given.

*  An idx >= width is undefined.*/

String* col_name(size_t idx)

/** Return type of column at idx. An idx >= width is undefined. */

char col_type(size_t idx)

/** Given a column name return its index, or -1. */

int col_idx(const char* name)

/** Given a row name return its index, or -1. */

int row_idx(const char* name)

/** The number of columns */

size_t width()

/** The number of rows */

size_t length()

};

/*****************************************************************************

* Fielder::

* A field vistor invoked by Row.

*/

class Fielder : public Object {

public:

/** Called before visiting a row, the argument is the row offset in the

dataframe. */

virtual void start(size_t r)

/** Called for fields of the argument's type with the value of the field. */

virtual void accept(bool b)

virtual void accept(float f)

virtual void accept(int i)

virtual void accept(String* s)

/** Called when all fields have been seen. */

virtual void done()

};

/*************************************************************************

* Row::

*

* This class represents a single row of data constructed according to a

* dataframe's schema. The purpose of this class is to make it easier to add

* read/write complete rows. Internally a dataframe hold data in columns.

* Rows have pointer equality.

*/

class Row : public Object {

public:

/** Build a row following a schema. */

Row(Schema& scm)

/** Setters: set the given column with the given value. Setting a column with

* a value of the wrong type is undefined. */

void set(size_t col, int val)

void set(size_t col, float val)

void set(size_t col, bool val)

/** The string is external. */

void set(size_t col, String* val)

/** Set/get the index of this row (ie. its position in the dataframe. This is

*  only used for informational purposes, unused otherwise */

void set_idx(size_t idx)

size_t get_idx()

/** Getters: get the value at the given column. If the column is not

* of the requested type, the result is undefined. */

int get_int(size_t col)

bool get_bool(size_t col)

float get_float(size_t col)

String* get_string(size_t col)

/** Number of fields in the row. */

size_t width()

/** Type of the field at the given position. An idx >= width is  undefined. */

char col_type(size_t idx)

/** Given a Fielder, visit every field of this row. The first argument is

* index of the row in the dataframe.

* Calling this method before the row's fields have been set is undefined. */

void visit(size_t idx, Fielder& f)

};

/*******************************************************************************

*  Rower::

*  An interface for iterating through each row of a data frame. The intent

*  is that this class should subclassed and the accept() method be given

*  a meaningful implementation. Rowers can be cloned for parallel execution.

*/

class Rower : public Object {

public:

/** This method is called once per row. The row object is on loan and

should not be retained as it is likely going to be reused in the next

call. The return value is used in filters to indicate that a row

should be kept. */

virtual bool accept(Row& r)

/** Once traversal of the data frame is complete the rowers that were

split off will be joined.  There will be one join per split. The

original object will be the last to be called join on. The join method

is reponsible for cleaning up memory. */

void join_delete(Rower* other)

};

/****************************************************************************

* DataFrame::

*

* A DataFrame is table composed of columns of equal length. Each column

* holds values of the same type (I, S, B, F). A dataframe has a schema that

* describes it.

*/

class DataFrame : public Object {

public:

/** Create a data frame with the same columns as the given df but with no rows or rownmaes */

DataFrame(DataFrame& df)

/** Create a data frame from a schema and columns. All columns are created

* empty. */

DataFrame(Schema& schema)

/** Returns the dataframe's schema. Modifying the schema after a dataframe

* has been created in undefined. */

Schema& get_schema()

/** Adds a column this dataframe, updates the schema, the new column

* is external, and appears as the last column of the dataframe, the

* name is optional and external. A nullptr colum is undefined. */

void add_column(Column* col, String* name)

/** Return the value at the given column and row. Accessing rows or

*  columns out of bounds, or request the wrong type is undefined.*/

int get_int(size_t col, size_t row)

bool get_bool(size_t col, size_t row)

float get_float(size_t col, size_t row)

String*  get_string(size_t col, size_t row)

/** Return the offset of the given column name or -1 if no such col. */

int get_col(String& col)

/** Return the offset of the given row name or -1 if no such row. */

int get_row(String& col)

/** Set the value at the given column and row to the given value.

* If the column is not  of the right type or the indices are out of

* bound, the result is undefined. */

void set(size_t col, size_t row, int val)

void set(size_t col, size_t row, bool val)

void set(size_t col, size_t row, float val)

void set(size_t col, size_t row, String* val)

/** Set the fields of the given row object with values from the columns at

* the given offset.  If the row is not form the same schema as the

* dataframe, results are undefined.

*/

void fill_row(size_t idx, Row& row)

/** Add a row at the end of this dataframe. The row is expected to have

*  the right schema and be filled with values, otherwise undedined.  */

void add_row(Row& row)

/** The number of rows in the dataframe. */

size_t nrows()

/** The number of columns in the dataframe.*/

size_t ncols()

/** Visit rows in order */

void map(Rower& r)

/** Create a new dataframe, constructed from rows for which the given Rower

* returned true from its accept method. */

DataFrame* filter(Rower& r)

/** Print the dataframe in SoR format to standard output. */

void print()

};