import os

DATA_DIR = os.path.abspath("../data")
MODEL_DIR = os.path.abspath("../models")
DATASET_DIR = f"{DATA_DIR}/datasets"
TRAINING_DIR = f"{DATA_DIR}/training"
VALIDATION_DIR = f"{DATA_DIR}/validation"
TESTING_DIR = f"{DATA_DIR}/testing"
TEMP_DIR = f"{DATA_DIR}/temp"
ORIGINAL_DATASET_FILES = ["news_sample.csv", "995,000_rows.csv"]
DATASET_FILES = ["news_sample.parquet", "995,000_rows.parquet"]

CHUNK_SIZE = 10000 # how many rows to work on at time, instead of loading the entire dataset into memory.
MAX_ROWS = -1 # only work with MAX_ROWS rows so testing things out isnt crazy slow. Set to -1 for infinite.
