bronku/file_sorting
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

initial runs

Browse source
This commit is contained in:
bronku 2025-11-17 08:26:23 +01:00
parent 9e9260c84d
commit 918c1cba78
18 changed files with 379 additions and 356 deletions
Download patch file Download diff file Expand all files Collapse all files

69
include/buffer.hpp
View file

@ -1,69 +0,0 @@
#pragma once
#include "record.hpp"
#include <stdexcept>
#include <vector>
class Buffer;
class SubBuffer {
private:
Record* data_;
size_t size_;
friend class Buffer;
SubBuffer(Record* data, size_t size)
: data_(data)
, size_(size)
{
}
public:
// so that i can create a vecotr of set size, without the .reserve(), and other stuff
SubBuffer()
{
}
// for std::sort support
using iterator = Record*;
iterator begin() { return data_; }
iterator end() { return data_ + size_; }
Record& operator[](size_t index)
{
return data_[index];
}
size_t size() const { return size_; }
};
class Buffer {
private:
std::vector<Record> storage_;
public:
explicit Buffer(size_t total_size)
: storage_(total_size)
{
}
std::vector<SubBuffer> divide(size_t n)
{
if (n == 0) {
throw std::invalid_argument("Cannot divide buffer into 0 pieces");
}
size_t total_size = storage_.size();
if (total_size % n != 0) {
throw std::invalid_argument("Size not divisible by n");
}
std::vector<SubBuffer> result(n);
size_t piece_size = total_size / n;
for (size_t i = 0; i < n; ++i) {
result[i] = SubBuffer(storage_.data() + i * piece_size, piece_size);
}
return result;
}
};

59
include/config.hpp
View file

@ -1,70 +1,17 @@
#pragma once
#include <getopt.h>
#include <iostream>
#include <stdexcept>
#include <string>
class Configuration {
public:
std::string input_file = "1.in";
std::string output_file = "1.in";
std::string directory = "tmp";
std::string output_file = "1.out";
std::string tmp_dir = "build/run";
bool generate_data = false;
bool evaluate_file = false;
int N = 100000; // number of records in a file
int n = 101; // number of buffers
int b = 10; // blocking factor
static Configuration parse_args(int argc, char** argv)
{
if (argc <= 1) {
std::cerr << "Please specify input arguments\n";
std::cerr << "Usage: " << argv[0] << " [-g] [-e] [-N count] [-i input] [-o output] [-d directory] [-n buffers] [-b block_size]\n";
throw std::invalid_argument("No input arguments");
}
Configuration config;
bool input_set = false;
int opt;
while ((opt = getopt(argc, argv, "i:o:d:geN:n:b:")) != -1) {
switch (opt) {
case 'i':
config.input_file = optarg;
input_set = true;
break;
case 'o':
config.output_file = optarg;
break;
case 'd':
config.directory = optarg;
break;
case 'g':
config.generate_data = true;
break;
case 'e':
config.evaluate_file = true;
break;
case 'N':
config.N = std::stoi(optarg);
break;
case 'n':
config.n = std::stoi(optarg);
break;
case 'b':
config.b = std::stoi(optarg);
break;
case '?':
throw std::invalid_argument("Invalid command line argument");
}
}
if (config.generate_data && input_set) {
throw std::invalid_argument("Cannot specify both -g and -i options");
}
if (config.generate_data && config.evaluate_file) {
throw std::invalid_argument("Cannot specify both -g and -e options");
}
return config;
}
Configuration(int argc, char** argv);
};

22
include/file_reader.hpp Normal file
View file

@ -0,0 +1,22 @@
#pragma once
#include "record.hpp"
#include <fstream>
#include <optional>
#include <span>
class FileReader {
private:
std::ifstream stream_;
std::vector<Record> buffer_;
size_t current_pos_;
size_t valid_items_;
size_t total_reads_;
void refill_buffer();
public:
FileReader(const std::string& filename, size_t buffer_size = 8);
std::optional<Record> read();
size_t read_chunk(std::span<Record>& chunk);
size_t total_reads() const;
};

23
include/file_writer.hpp Normal file
View file

@ -0,0 +1,23 @@
#pragma once
#include "record.hpp"
#include <fstream>
#include <span>
class FileWriter {
private:
std::ofstream stream_;
std::vector<Record> buffer_;
size_t current_pos_;
size_t total_writes_;
void write_buffer();
public:
FileWriter(const std::string& filename, size_t buffer_size = 8);
~FileWriter();
void write(const Record& record);
void write_chunk(std::span<const Record>& chunk);
size_t total_writes() const;
};

29
include/merge_sorter.hpp Normal file
View file

@ -0,0 +1,29 @@
#pragma once
#include "file_reader.hpp"
#include "file_writer.hpp"
#include "record.hpp"
#include <vector>
class MergeSorter {
private:
std::vector<Record> buffer_;
const std::string& tmp_dir_;
size_t buffer_rows = 0;
size_t disk_reads_ = 0;
size_t disk_writes_ = 0;
size_t phases_ = 0;
size_t create_initial_runs(const std::string& input_file);
void merge_runs(const std::vector<std::string>& input_files,
const std::string& output_file);
void perform_k_way_merge(std::vector<std::unique_ptr<FileReader>>& readers,
FileWriter& writer);
public:
MergeSorter(size_t buffer_rows, size_t buffer_cols, const std::string& tmp_dir);
bool sort_file(const std::string& input_file, const std::string& output_file);
size_t disk_reads() const;
size_t disk_writes() const;
size_t phases() const;
};

79
include/reader.hpp
View file

@ -1,79 +0,0 @@
// #todo maybe overload >> operators insted of read()
#pragma once
#include "record.hpp"
#include <istream>
#include <optional>
#include <vector>
class Reader {
private:
std::vector<Record> read_buffer;
std::istream& input_stream;
size_t buffer_size;
size_t total_reads_count;
int last_element_idx; // last valid element in buffer
int current_idx; // current reading position
// returns true if at least one record was returned
bool refill_buffer()
{
last_element_idx = -1;
current_idx = 0;
for (size_t i = 0; i < buffer_size; i++) {
if (!(input_stream >> read_buffer[i])) {
break;
}
last_element_idx = i;
}
return last_element_idx >= 0;
}
public:
explicit Reader(std::istream& input_stream, size_t buffer_size = 8)
: read_buffer(buffer_size)
, input_stream(input_stream)
, buffer_size(buffer_size)
, total_reads_count(0)
, last_element_idx(-1)
, current_idx(0)
{
}
std::optional<Record> read()
{
if (current_idx > last_element_idx) {
bool read_new_items = refill_buffer();
if (!read_new_items) {
return std::nullopt;
}
}
Record record = read_buffer[current_idx];
current_idx++;
total_reads_count++;
return record;
}
size_t total_reads() const
{
return total_reads_count;
}
// #todo remove if not used in the end
// bool has_more() const
// {
// return current_idx <= last_element_idx || !input_stream.eof();
// }
// void reset()
// {
// input_stream.clear();
// input_stream.seekg(0);
// last_element_idx = -1;
// current_idx = 0;
// total_reads_count = 0;
// }
};

48
include/record.hpp
View file

@ -1,53 +1,17 @@
#pragma once
#include <array>
#include <iostream>
#include <random>
class Record {
public:
std::array<int, 5> a;
int x;
int evaluate() const
{
int out = 0;
int x_n = 1;
for (int i = 0; i < 5; i++) {
out += a[i] * x_n;
x_n *= x;
}
return out;
}
int evaluate() const;
bool operator<(const Record& other) const;
bool operator<(const Record& other) const
{
return evaluate() < other.evaluate();
}
friend std::ostream& operator<<(std::ostream& os, const Record& r);
friend std::istream& operator>>(std::istream& is, Record& r);
friend std::ostream& operator<<(std::ostream& os, const Record& r)
{
os << r.a[0] << ' ' << r.a[1] << ' ' << r.a[2] << ' '
<< r.a[3] << ' ' << r.a[4] << ' ' << r.x;
return os;
}
friend std::istream& operator>>(std::istream& is, Record& r)
{
is >> r.a[0] >> r.a[1] >> r.a[2] >> r.a[3] >> r.a[4] >> r.x;
return is;
}
static Record random()
{
static std::random_device rd;
static std::mt19937 gen(rd());
static std::uniform_int_distribution<int> dist;
Record r;
for (int& val : r.a) {
val = dist(gen);
}
r.x = dist(gen);
return r;
}
};
static Record random();
};

63
include/writer.hpp
View file

@ -1,63 +0,0 @@
// #todo maybe overload << operators insted of write()
#pragma once
#include "record.hpp"
#include <ostream>
#include <vector>
class Writer {
private:
std::vector<Record> write_buffer;
std::ostream& output_stream;
size_t buffer_size;
size_t total_writes_count;
size_t current_index;
// returns true on success
// doesn't really flush for performace reasons
// could be changed to really flush to monitor the impact of amount of writes
bool flush_buffer()
{
for (size_t i = 0; i < current_index; ++i) {
if (!(output_stream << write_buffer[i] << " ")) {
return false;
}
}
current_index = 0;
return true;
}
public:
explicit Writer(std::ostream& output_stream, size_t buffer_size = 8)
: write_buffer(buffer_size)
, output_stream(output_stream)
, buffer_size(buffer_size)
, total_writes_count(0)
, current_index(0)
{
}
~Writer()
{
flush_buffer();
}
bool write(const Record& record)
{
write_buffer[current_index] = record;
current_index++;
total_writes_count++;
if (current_index >= buffer_size) {
return flush_buffer();
}
return true;
}
size_t total_writes() const
{
return total_writes_count;
}
};