diff --git a/docker-compose.yml b/docker-compose.yml index b00b8a6..507ef6c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: image: 'ecotaxa/ecopart_back:latest' env_file: .env volumes: - - ./sqlite_db:/src/sqlite_db + - ./data_storage:/src/data_storage - type: bind source: ./.env target: /src/.env diff --git a/empty.env b/empty.env index a1d3742..7664234 100644 --- a/empty.env +++ b/empty.env @@ -10,7 +10,8 @@ PORT_LOCAL = {$Your local port} BASE_URL_PUBLIC = {$Your public url} PORT_PUBLIC = {$Your public port} DBSOURCE_NAME = {$YourDBname.db} -DBSOURCE_FOLDER = sqlite_db/ +DBSOURCE_FOLDER = data_storage/sqlite_db/ +DATA_STORAGE_FOLDER = data_storage/ # Replace with your SMTP server MAIL_HOST = {$smtp.example.com} diff --git a/src/data/data-sources/sqlite/sqlite-task-data-source.ts b/src/data/data-sources/sqlite/sqlite-task-data-source.ts index 5bcb6c5..07b0ae7 100644 --- a/src/data/data-sources/sqlite/sqlite-task-data-source.ts +++ b/src/data/data-sources/sqlite/sqlite-task-data-source.ts @@ -1,7 +1,7 @@ import { TaskDataSource } from "../../interfaces/data-sources/task-data-source"; import { SQLiteDatabaseWrapper } from "../../interfaces/data-sources/database-wrapper"; import { PreparedSearchOptions, SearchResult } from "../../../domain/entities/search"; -import { PrivateTaskRequestModel, PrivateTaskRequestCreationModel, TaskResponseModel, TaskTypeResponseModel, TaskStatusResponseModel } from "../../../domain/entities/task"; +import { PrivateTaskRequestModel, PrivateTaskRequestCreationModel, TaskResponseModel, TaskTypeResponseModel, TaskStatusResponseModel, PrivateTaskUpdateModel } from "../../../domain/entities/task"; import { UserRequestModel } from "../../../domain/entities/user"; export class SQLiteTaskDataSource implements TaskDataSource { @@ -37,7 +37,15 @@ export class SQLiteTaskDataSource implements TaskDataSource { else { // Insert default task_status - const sql_admin = "INSERT OR IGNORE INTO task_status (task_status_label) VALUES ('PENDING', 'VALIDATING', 'RUNNING', 'WAITING_FO_RESPONSE', 'DONE', 'ERROR');"; + const sql_admin = ` + INSERT OR IGNORE INTO task_status (task_status_label) + VALUES + ('PENDING'), + ('VALIDATING'), + ('RUNNING'), + ('WAITING_FOR_RESPONSE'), + ('DONE'), + ('ERROR');`; db_tables.run(sql_admin, [], function (err: Error | null) { if (err) { @@ -66,8 +74,16 @@ export class SQLiteTaskDataSource implements TaskDataSource { else { // Insert default task_type - const sql_admin = "INSERT OR IGNORE INTO task_type (task_type_label) VALUES ('EXPORT', 'DELETE', 'UPDATE', 'IMPORT', 'IMPORT_CTD', 'IMPORT_ECO_TAXA');"; - + const sql_admin = ` + INSERT OR IGNORE INTO task_type (task_type_label) + VALUES + ('EXPORT'), + ('DELETE'), + ('UPDATE'), + ('IMPORT'), + ('IMPORT_CTD'), + ('IMPORT_ECO_TAXA'); + `; db_tables.run(sql_admin, [], function (err: Error | null) { if (err) { console.log("DB error--", err); @@ -99,8 +115,8 @@ export class SQLiteTaskDataSource implements TaskDataSource { task_end_date TIMESTAMP, FOREIGN KEY (task_type_id) REFERENCES task_type(task_type_id), FOREIGN KEY (task_status_id) REFERENCES task_status(task_status_id), - FOREIGN KEY (task_owner_id) REFERENCES user(task_owner_id) ON DELETE CASCADE, - FOREIGN KEY (task_project_id) REFERENCES project(task_project_id) ON DELETE CASCADE + FOREIGN KEY (task_owner_id) REFERENCES user(user_id) ON DELETE CASCADE, + FOREIGN KEY (task_project_id) REFERENCES project(project_id) ON DELETE CASCADE );` // Run the SQL query to create the table @@ -271,7 +287,7 @@ export class SQLiteTaskDataSource implements TaskDataSource { task_owner_id: row.task_owner_id, task_owner: row.user_first_name + " " + row.user_last_name + " (" + row.email + ")", // Doe John (john.doe@mail.com) task_project_id: row.task_project_id, - task_file_path: row.task_log_file_path, + task_log_file_path: row.task_log_file_path, task_progress_pct: row.task_progress_pct, task_progress_msg: row.task_progress_msg, task_params: row.task_params, @@ -335,7 +351,7 @@ export class SQLiteTaskDataSource implements TaskDataSource { task_owner_id: row.task_owner_id, task_owner: row.user_first_name + " " + row.user_last_name + " (" + row.email + ")", // Doe John (john.doe@mail.com) task_project_id: row.task_project_id, - task_file_path: row.task_log_file_path, + task_log_file_path: row.task_log_file_path, task_progress_pct: row.task_progress_pct, task_progress_msg: row.task_progress_msg, task_params: row.task_params, @@ -543,5 +559,34 @@ export class SQLiteTaskDataSource implements TaskDataSource { }); }) } -} + // Update One task + // Returns the number of lines updates + async updateOne(task: PrivateTaskUpdateModel): Promise { + const { task_id, ...taskData } = task; // Destructure the project object + const params: any[] = [] + let placeholders: string = "" + // generate sql and params + for (const [key, value] of Object.entries(taskData)) { + params.push(value) + placeholders = placeholders + key + "=(?)," + } + // remove last , + placeholders = placeholders.slice(0, -1); + // add task_id to params + params.push(task_id) + + // form final sql + const sql = `UPDATE task SET ` + placeholders + ` WHERE task_id=(?);`; + return await new Promise((resolve, reject) => { + this.db.run(sql, params, function (err) { + if (err) { + reject(err); + } else { + const result = this.changes; + resolve(result); + } + }); + }) + } +} \ No newline at end of file diff --git a/src/data/interfaces/data-sources/task-data-source.ts b/src/data/interfaces/data-sources/task-data-source.ts index 7f49ee6..fe49326 100644 --- a/src/data/interfaces/data-sources/task-data-source.ts +++ b/src/data/interfaces/data-sources/task-data-source.ts @@ -1,4 +1,4 @@ -import { TaskResponseModel, PrivateTaskRequestCreationModel, PrivateTaskRequestModel, TaskTypeResponseModel, TaskStatusResponseModel } from "../../../domain/entities/task"; +import { TaskResponseModel, PrivateTaskRequestCreationModel, PrivateTaskRequestModel, TaskTypeResponseModel, TaskStatusResponseModel, PrivateTaskUpdateModel } from "../../../domain/entities/task"; import { PreparedSearchOptions, SearchResult } from "../../../domain/entities/search"; import { UserRequestModel } from "../../../domain/entities/user"; @@ -11,4 +11,5 @@ export interface TaskDataSource { getOne(task: PrivateTaskRequestModel): Promise; getAllType(options: PreparedSearchOptions): Promise> getAllStatus(options: PreparedSearchOptions): Promise> + updateOne(task: PrivateTaskUpdateModel): Promise } \ No newline at end of file diff --git a/src/domain/entities/task.ts b/src/domain/entities/task.ts index 7eb9bf0..b68f0f7 100644 --- a/src/domain/entities/task.ts +++ b/src/domain/entities/task.ts @@ -25,15 +25,15 @@ export enum TaskAction { /* CREATION */ export interface PublicTaskRequestCreationModel { - task_type_id: TaskType; - task_status_id: TasksStatus; - task_owner_id: number; // task owner : string in a public version + task_type: TaskType; + task_status: TasksStatus; + task_owner_id: number; task_project_id?: number; task_params: object; } export interface PrivateTaskRequestCreationModel { - task_type_id: TaskType; - task_status_id: TasksStatus; + task_type_id: number; //TaskType; + task_status_id: number; //TasksStatus; task_owner_id: number; // task owner : string in a public version task_project_id?: number; @@ -116,4 +116,31 @@ export interface PublicTaskRequestModel extends PrivateTaskRequestModel { task_status?: string; task_owner?: string; } +export interface PrivateTaskUpdateModel { + task_id: number; + task_type_id?: number; + task_type?: string; + task_status_id?: number; + task_status?: string; + task_owner_id?: number; + task_owner?: string; + task_project_id?: number; + task_params?: object; + task_creation_date?: string; + task_start_date?: string; + task_end_date?: string; + task_log_file_path?: string; + + task_progress_pct?: number; + task_progress_msg?: string; + task_result?: string; + task_error?: string; + task_question?: string; + task_reply?: string; + task_step?: string; +} +// export interface PublicTaskUpdateModel { +// task_id: number; +// task_reply: string; +// } \ No newline at end of file diff --git a/src/domain/interfaces/repositories/sample-repository.ts b/src/domain/interfaces/repositories/sample-repository.ts index 97ac927..74facb1 100644 --- a/src/domain/interfaces/repositories/sample-repository.ts +++ b/src/domain/interfaces/repositories/sample-repository.ts @@ -3,4 +3,5 @@ import { PublicSampleResponseModel } from "../../entities/sample"; export interface SampleRepository { ensureFolderExists(root_folder_path: string): Promise; listImportableSamples(root_folder_path: string, instrument_model: string): Promise; + copySamplesToImportFolder(source_folder: string, dest_folder: string, samples_names_to_import: string[]): Promise } \ No newline at end of file diff --git a/src/domain/interfaces/repositories/task-repository.ts b/src/domain/interfaces/repositories/task-repository.ts index ddb4612..f58eb9e 100644 --- a/src/domain/interfaces/repositories/task-repository.ts +++ b/src/domain/interfaces/repositories/task-repository.ts @@ -4,19 +4,23 @@ // import { PreparedSearchOptions, SearchResult } from "../../entities/search"; import { PreparedSearchOptions, SearchResult } from "../../entities/search"; -import { PrivateTaskRequestModel, TaskResponseModel, TaskStatusResponseModel, TaskTypeResponseModel } from "../../entities/task"; +import { PublicTaskRequestCreationModel, PrivateTaskRequestModel, TaskResponseModel, TaskStatusResponseModel, TaskTypeResponseModel, PublicTaskRequestModel } from "../../entities/task"; import { UserRequestModel } from "../../entities/user"; export interface TaskRepository { - getOneTask(task: PrivateTaskRequestModel): Promise + getOneTask(task: PrivateTaskRequestModel): Promise; + startTask(task: PublicTaskRequestModel): Promise; + finishTask(task: PublicTaskRequestModel): Promise; + updateTaskProgress(task: PublicTaskRequestModel, progress_pct: number, progress_msg: string): Promise; // formatTaskRequestCreationModel(public_task: PublicTaskRequestCreationModel, instrument: InstrumentModelResponseModel): TaskRequestCreationModel; // standardUpdateTask(task_to_update: TaskUpdateModel): Promise; - // createTask(task: TaskRequestCreationModel): Promise; + createTask(task: PublicTaskRequestCreationModel): Promise; getTask(task: PrivateTaskRequestModel): Promise; deleteTask(task: PrivateTaskRequestModel): Promise; standardGetTasks(options: PreparedSearchOptions): Promise>; standardGetTaskType(options: PreparedSearchOptions): Promise> standardGetTaskStatus(options: PreparedSearchOptions): Promise>; getTasksByUser(user: UserRequestModel): Promise; - getLogFileTask(task_id: number): Promise + getLogFileTask(task_id: number): Promise; + failedTask(task_id: number, error: Error): Promise; } diff --git a/src/domain/interfaces/use-cases/sample/import-samples.ts b/src/domain/interfaces/use-cases/sample/import-samples.ts new file mode 100644 index 0000000..530e044 --- /dev/null +++ b/src/domain/interfaces/use-cases/sample/import-samples.ts @@ -0,0 +1,6 @@ +import { TaskResponseModel } from "../../../entities/task"; +import { UserUpdateModel } from "../../../entities/user"; + +export interface ImportSamplesUseCase { + execute(current_user: UserUpdateModel, project_id: number, samples_names: string[]): Promise; +} \ No newline at end of file diff --git a/src/domain/repositories/sample-repository.ts b/src/domain/repositories/sample-repository.ts index 68ab3b2..3e89529 100644 --- a/src/domain/repositories/sample-repository.ts +++ b/src/domain/repositories/sample-repository.ts @@ -30,7 +30,6 @@ export class SampleRepositoryImpl implements SampleRepository { try { await fs.access(folderPath); - console.log('Folder exists'); } catch (error) { throw new Error(`Folder does not exist at path: ${folderPath}`); } @@ -55,6 +54,7 @@ export class SampleRepositoryImpl implements SampleRepository { return samples; } + // Function to setup samples async setupSamples(meta_header_samples: HeaderSampleModel[], samples: string[], folder: string): Promise { // flag qc samples to flase if not in both lists, and add qc message @@ -98,7 +98,6 @@ export class SampleRepositoryImpl implements SampleRepository { } getSampleFromHeaderLine(line: string): HeaderSampleModel { - console.log('line', line); const fields = line.split(';'); const sample: HeaderSampleModel = { @@ -163,6 +162,43 @@ export class SampleRepositoryImpl implements SampleRepository { return samples; } + // This needs to be inside an async function to use await + async ensureSampleFolderDoNotExists(samples_names_to_import: string[], dest_folder: string): Promise { + // Ensure that none of the sample folders already exist + for (const sample of samples_names_to_import) { + const destPath = path.join(dest_folder, sample); + try { + await fs.access(destPath); + throw new Error(`Sample folder already exists: ${destPath}`); + } catch (error) { + if (error.code === 'ENOENT') { + // Do nothing, the folder does not exist + } else { + // Throw other types of errors, e.g., permission issues + throw error; + } + } + } + } + + async copySamplesToImportFolder(source_folder: string, dest_folder: string, samples_names_to_import: string[]): Promise { + + const base_folder = path.join(__dirname, '..', '..', '..'); + // Ensure that non of the samples folder already exists + await this.ensureSampleFolderDoNotExists(samples_names_to_import, path.join(base_folder, dest_folder)); + + // Ensure destination folder exists + await fs.mkdir(path.join(base_folder, dest_folder), { recursive: true }); + + // Iterate over each sample name and copy it + for (const sample of samples_names_to_import) { + const sourcePath = path.join(base_folder, source_folder, sample); + const destPath = path.join(base_folder, dest_folder, sample); + + // Copy the sample folder recurcively from source to destination + await fs.cp(sourcePath, destPath, { recursive: true, errorOnExist: true }); + } + } diff --git a/src/domain/repositories/task-repository.ts b/src/domain/repositories/task-repository.ts index 1a0cb10..1243a84 100644 --- a/src/domain/repositories/task-repository.ts +++ b/src/domain/repositories/task-repository.ts @@ -1,30 +1,130 @@ +import path from "path"; import { TaskDataSource } from "../../data/interfaces/data-sources/task-data-source"; +import { FsWrapper } from "../../infra/files/fs-wrapper"; import { PreparedSearchOptions, SearchResult } from "../entities/search"; // import { InstrumentModelResponseModel } from "../entities/instrument_model"; // import { PublicPrivilege } from "../entities/privilege"; // import { TaskRequestCreationModel, PrivateTaskRequestModel, TaskUpdateModel, TaskResponseModel, PublicTaskResponseModel, PublicTaskRequestCreationModel } from "../entities/task"; +import { PrivateTaskRequestCreationModel, PublicTaskRequestCreationModel, PublicTaskRequestModel, TasksStatus } from "../entities/task"; import { PrivateTaskRequestModel, TaskResponseModel, TaskStatusResponseModel, TaskTypeResponseModel } from "../entities/task"; import { UserRequestModel } from "../entities/user"; // import { PreparedSearchOptions, SearchResult } from "../entities/search"; import { TaskRepository } from "../interfaces/repositories/task-repository"; -import fs from 'node:fs/promises'; export class TaskRepositoryImpl implements TaskRepository { taskDataSource: TaskDataSource + fs: FsWrapper + DATA_STORAGE_FOLDER: string // TODO move to a search repository order_by_allow_params: string[] = ["asc", "desc"] filter_operator_allow_params: string[] = ["=", ">", "<", ">=", "<=", "<>", "IN", "LIKE"] - constructor(taskDataSource: TaskDataSource) { + constructor(taskDataSource: TaskDataSource, fs: FsWrapper, DATA_STORAGE_FOLDER: string) { this.taskDataSource = taskDataSource + this.fs = fs + this.DATA_STORAGE_FOLDER = DATA_STORAGE_FOLDER } - // async createTask(task: TaskRequestCreationModel): Promise { - // const result = await this.taskDataSource.create(task) - // return result; - // } + async createTask(task: PublicTaskRequestCreationModel): Promise { + const type_options: PreparedSearchOptions = { + sort_by: [{ sort_by: "task_type_id", order_by: "asc" }], + filter: [{ field: "task_type_label", operator: "=", value: task.task_type }], + page: 1, + limit: 1 + } + const status_options: PreparedSearchOptions = { + sort_by: [{ sort_by: "task_status_id", order_by: "asc" }], + filter: [{ field: "task_status_label", operator: "=", value: task.task_status }], + page: 1, + limit: 1 + } + // Transform the public task to a private task + const private_task: PrivateTaskRequestCreationModel = { + task_type_id: await this.taskDataSource.getAllType(type_options).then(result => { + if (result.items.length === 0) { + throw new Error("Task type not found") + } + return result.items[0].task_type_id + }), + task_status_id: await this.taskDataSource.getAllStatus(status_options).then(result => { + if (result.items.length === 0) { + throw new Error("Task status not found") + } + return result.items[0].task_status_id + }), + task_owner_id: task.task_owner_id, + task_project_id: task.task_project_id, + task_log_file_path: "", + task_params: JSON.stringify(task.task_params) + } + const result = await this.taskDataSource.create(private_task) + + //create log file based on created task_id + const log_file_path = path.join(__dirname, '..', '..', '..', this.DATA_STORAGE_FOLDER, "tasks_log", `task_${result}.log`) + + // create log file + try { + await this.logMessage(log_file_path, "Task created") + await this.logMessage(log_file_path, "Task params: " + JSON.stringify(task.task_params)) + } catch (err) { + console.log(err); + throw new Error("Cannot create log file"); + } + // update task with log file path + await this.taskDataSource.updateOne({ task_id: result, task_log_file_path: log_file_path }) + + return result; + } + + async startTask(task: PublicTaskRequestModel): Promise { + + const task_to_start = await this.taskDataSource.getOne({ task_id: task.task_id }) + if (!task_to_start) { + throw new Error("Task not found") + } + + // Update the task status to running + await this.statusManager({ task_id: task_to_start.task_id }, TasksStatus.Running) + + // appendFile to log file that task is running + await this.logMessage(task_to_start.task_log_file_path, "Task is running") + } + + async finishTask(task: PublicTaskRequestModel): Promise { + const task_to_finish = await this.taskDataSource.getOne({ task_id: task.task_id }) + if (!task_to_finish) { + throw new Error("Task not found") + } + + // Update the task status to done + await this.statusManager({ task_id: task_to_finish.task_id }, TasksStatus.Done) + + // Update the task progress to 100% and add a message + await this.updateTaskProgress(task, 100, "Task is done sucessfilly") + + // appendFile to log file that task is done + await this.logMessage(task_to_finish.task_log_file_path, "Task is done sucessfilly") + } + + async updateTaskProgress(task: PublicTaskRequestModel, progress_pct: number, progress_msg: string): Promise { + const task_to_update = await this.ensureTaskExists(task.task_id || 0) + + // Update the task progress + await this.taskDataSource.updateOne({ task_id: task_to_update.task_id, task_progress_pct: progress_pct, task_progress_msg: progress_msg }) + + // appendFile to log file the progress + await this.logMessage(task_to_update.task_log_file_path, `Task progress: ${progress_pct}% - ${progress_msg}`) + } + + async ensureTaskExists(task_id: number): Promise { + const task_to_update = await this.taskDataSource.getOne({ task_id: task_id }) + if (!task_to_update) { + throw new Error("Task not found") + } + return task_to_update + } async getTask(task: PrivateTaskRequestModel): Promise { const result = await this.taskDataSource.getOne(task) @@ -220,6 +320,7 @@ export class TaskRepositoryImpl implements TaskRepository { } // Get One Task async getOneTask(task: PrivateTaskRequestModel): Promise { + console.log("getOneTask", task) return await this.taskDataSource.getOne(task); } async getTasksByUser(user: UserRequestModel): Promise { @@ -237,7 +338,7 @@ export class TaskRepositoryImpl implements TaskRepository { } // read log file try { - const data = await fs.readFile(task_log_file_path, { encoding: 'utf8' }); + const data = await this.fs.readFile(task_log_file_path, { encoding: 'utf8' }); return data; } catch (err) { // if error, throw an error @@ -245,4 +346,77 @@ export class TaskRepositoryImpl implements TaskRepository { throw new Error("Cannot read log file"); } } + + async failedTask(task_id: number, error: Error): Promise { + this.getTask({ task_id: task_id }).then(async task => { + if (!task) throw new Error("Task not found") + + // log error in log file + await this.logMessage(task.task_log_file_path, `Task failed with error: ${error.message}`); + + // Update the task error message + await this.taskDataSource.updateOne({ task_id: task_id, task_error: error.message }) + + // Update the task status to error + this.statusManager({ + task_id: task_id + }, TasksStatus.Error) + + }) + } + + async logMessage(task_log_file_path: string | undefined, message: string): Promise { + if (!task_log_file_path) throw new Error("No log file path found for this task") + const log_file_path = task_log_file_path; + if (log_file_path) await this.fs.appendFile(log_file_path, new Date().toISOString() + " " + message + "\n"); + } + // Define allowed transitions between statuses + private allowedTransitions: { [key in TasksStatus]: TasksStatus[] } = { + [TasksStatus.Pending]: [TasksStatus.Running], + [TasksStatus.Running]: [TasksStatus.Waiting_for_response, TasksStatus.Done, TasksStatus.Error], + [TasksStatus.Waiting_for_response]: [TasksStatus.Running], + [TasksStatus.Done]: [], + [TasksStatus.Error]: [], + }; + + async statusManager(task: PublicTaskRequestModel, status: TasksStatus): Promise { + // Retrieve the task to update + const task_to_update = await this.taskDataSource.getOne({ task_id: task.task_id }); + + // Task not found + if (!task_to_update) { + throw new Error("Task not found"); + } + + // Check if the task is already in the requested status + if (task_to_update.task_status === status) { + throw new Error("Task is already in this status"); + } + + // Check if the transition to the requested status is allowed + const allowedStatuses = this.allowedTransitions[task_to_update.task_status as TasksStatus]; + if (!allowedStatuses.includes(status)) { + throw new Error(`Cannot change status from ${task_to_update.task_status} to ${status}`); + } + + // Retrieve the status id of the requested status + const transition_status_id = await this.taskDataSource.getAllStatus({ + sort_by: [{ sort_by: "task_status_id", order_by: "asc" }], + filter: [{ field: "task_status_label", operator: "=", value: status }], + page: 1, + limit: 1 + }).then(result => { + if (result.items.length === 0) { + throw new Error("Task status not found") + } + return result.items[0].task_status_id + }); + + // Update the task status if valid + await this.taskDataSource.updateOne({ task_id: task_to_update.task_id, task_status_id: transition_status_id }); + + + // Logging task status update in the log file + await this.logMessage(task_to_update.task_log_file_path, `Task status updated from ${task_to_update.task_status} to ${status}`); + } } \ No newline at end of file diff --git a/src/domain/use-cases/sample/import-samples.ts b/src/domain/use-cases/sample/import-samples.ts new file mode 100644 index 0000000..7e9afee --- /dev/null +++ b/src/domain/use-cases/sample/import-samples.ts @@ -0,0 +1,165 @@ + +import { PublicSampleResponseModel } from "../../entities/sample"; +import { UserUpdateModel } from "../../entities/user"; +import { PrivilegeRepository } from "../../interfaces/repositories/privilege-repository"; +import { SampleRepository } from "../../interfaces/repositories/sample-repository"; +import { ProjectRepository } from "../../interfaces/repositories/project-repository"; +import { UserRepository } from "../../interfaces/repositories/user-repository"; +import { TaskRepository } from "../../interfaces/repositories/task-repository"; + +import { ImportSamplesUseCase } from "../../interfaces/use-cases/sample/import-samples"; +import { ProjectResponseModel } from "../../entities/project"; +import { PublicTaskRequestCreationModel, TaskResponseModel, TasksStatus, TaskType } from "../../entities/task"; +import path from "path"; + +export class ImportSamples implements ImportSamplesUseCase { + sampleRepository: SampleRepository + userRepository: UserRepository + privilegeRepository: PrivilegeRepository + projectRepository: ProjectRepository + taskRepository: TaskRepository + DATA_STORAGE_FS_STORAGE: string + + constructor(sampleRepository: SampleRepository, userRepository: UserRepository, privilegeRepository: PrivilegeRepository, projectRepository: ProjectRepository, taskRepository: TaskRepository, DATA_STORAGE_FS_STORAGE: string) { + this.sampleRepository = sampleRepository + this.userRepository = userRepository + this.privilegeRepository = privilegeRepository + this.projectRepository = projectRepository + this.taskRepository = taskRepository + this.DATA_STORAGE_FS_STORAGE = DATA_STORAGE_FS_STORAGE + } + + async execute(current_user: UserUpdateModel, project_id: number, samples_names_to_import: string[]): Promise { + // Ensure the user is valid and can be used + await this.userRepository.ensureUserCanBeUsed(current_user.user_id); + + // Ensure the current user has permission to get the project importable samples + await this.ensureUserCanGet(current_user, project_id); + + const project: ProjectResponseModel = await this.getProjectIfExist(project_id); + + const samples = await this.listImportableSamples(project); + + // Check that asked samples are in the importable list of samples + this.ensureSamplesAreImportables(samples, samples_names_to_import); + + // create a task to import samples + const task_id = await this.createImportSamplesTask(current_user, project, samples_names_to_import); + + // get the task + const task = await this.taskRepository.getOneTask({ task_id: task_id }); + if (!task) { + throw new Error("Cannot find task"); + } + + // start the task + this.startImportTask(task, samples_names_to_import, project.instrument_model, project); + + return task; + } + ensureSamplesAreImportables(samples: PublicSampleResponseModel[], samples_names_to_import: string[]) { + this.ensureSamplesAreBothInHeadersAndInRawData(samples, samples_names_to_import); + this.ensureSamplesAreNotAlreadyImported(samples_names_to_import); + } + + ensureSamplesAreBothInHeadersAndInRawData(samples: PublicSampleResponseModel[], samples_names_to_import: string[]) { + const samples_names_set = new Set(samples.map(sample => sample.sample_name)); + + const missing_samples = samples_names_to_import.filter(sample_id => !samples_names_set.has(sample_id)); + + if (missing_samples.length > 0) { + throw new Error("Samples not importable: " + missing_samples.join(", ")); + } + } + + ensureSamplesAreNotAlreadyImported(samples_names_to_import: string[]) { + //TODO + console.log("TODO: ensureSamplesAreNotAlreadyImported please do an import update", samples_names_to_import); + } + + createImportSamplesTask(current_user: UserUpdateModel, project: ProjectResponseModel, samples: string[]) { + const task: PublicTaskRequestCreationModel = { + task_type: TaskType.Import, + task_status: TasksStatus.Pending, + task_owner_id: current_user.user_id, + task_project_id: project.project_id, + task_params: { samples: samples } + } + return this.taskRepository.createTask(task); + + } + + private async listImportableSamples(project: ProjectResponseModel): Promise { + await this.sampleRepository.ensureFolderExists(project.root_folder_path); + const samples = await this.sampleRepository.listImportableSamples(project.root_folder_path, project.instrument_model); + // Ensure the task to get exists + if (!samples) { throw new Error("No samples to import"); } + + return samples; + } + + private async getProjectIfExist(project_id: number): Promise { + const project = await this.projectRepository.getProject({ project_id: project_id }); + if (!project) { + throw new Error("Cannot find project"); + } + return project; + } + + private async ensureUserCanGet(current_user: UserUpdateModel, project_id: number): Promise { + const userIsAdmin = await this.userRepository.isAdmin(current_user.user_id); + const userHasPrivilege = await this.privilegeRepository.isGranted({ + user_id: current_user.user_id, + project_id: project_id + }); + if (!userIsAdmin && !userHasPrivilege) { + throw new Error("Logged user cannot list importable samples in this project"); + } + } + + private async startImportTask(task: TaskResponseModel, samples_names_to_import: string[], instrument_model: string, project: ProjectResponseModel) { + const task_id = task.task_id; + try { + await this.taskRepository.startTask({ task_id: task_id }); + + // 1/4 Do validation before importing + //TODO LATER + + // 2/4 Copy source files to hiden project folder + await this.copySourcesToProjectFolder(task_id, samples_names_to_import, instrument_model, project); + + // 3/4 Import samples + //await this.sampleRepository.importSamples(task_id, project.project_id, samples_names_to_import); + + // 4/4 generate qc report by samples + //TODO LATER but we can already et the qc flag to un validated + + // finish task + await this.taskRepository.finishTask({ task_id: task_id }); + } catch (error) { + this.taskRepository.failedTask(task_id, error); + } + + } + + async copySourcesToProjectFolder(task_id: number, samples_names_to_import: string[], instrument_model: string, project: ProjectResponseModel) { + const dest_folder = path.join(this.DATA_STORAGE_FS_STORAGE, `${project.project_id}`); + const root_folder_path = project.root_folder_path; + let source_folder; + + if (instrument_model.startsWith('UVP6')) { + source_folder = path.join(root_folder_path, 'ecodata'); + } else if (instrument_model.startsWith('UVP5')) { + source_folder = path.join(root_folder_path, 'work'); + } else { + throw new Error("Unknown instrument model"); + } + await this.sampleRepository.copySamplesToImportFolder(source_folder, dest_folder, samples_names_to_import); + await this.taskRepository.updateTaskProgress({ task_id: task_id }, 50, "Step 2/4 sample folders copied"); + + } + + // async importSamples(task_id: number, project_id: number, samples_names_to_import: string[]) { + + // } +} \ No newline at end of file diff --git a/src/domain/use-cases/sample/list-importable-samples.ts b/src/domain/use-cases/sample/list-importable-samples.ts index 6f39dce..58e9fe8 100644 --- a/src/domain/use-cases/sample/list-importable-samples.ts +++ b/src/domain/use-cases/sample/list-importable-samples.ts @@ -33,7 +33,7 @@ export class ListImportableSamples implements ListImportableSamplesUseCase { const samples = await this.listImportableSamples(project); // Ensure the task to get exists - if (!samples) { throw new Error("Cannot find samples"); } + if (!samples) { throw new Error("No samples to import"); } return samples; } @@ -53,16 +53,11 @@ export class ListImportableSamples implements ListImportableSamplesUseCase { } private async ensureUserCanGet(current_user: UserUpdateModel, project_id: number): Promise { - console.log("project_id : ", project_id); - console.log("current_user.user_id : ", current_user.user_id); const userIsAdmin = await this.userRepository.isAdmin(current_user.user_id); - console.log("userIsAdmin : ", userIsAdmin); const userHasPrivilege = await this.privilegeRepository.isGranted({ user_id: current_user.user_id, project_id: project_id }); - console.log("userHasPrivilege : ", userHasPrivilege); - if (!userIsAdmin && !userHasPrivilege) { throw new Error("Logged user cannot list importable samples in this project"); } diff --git a/src/domain/use-cases/task/delete-tasks.ts b/src/domain/use-cases/task/delete-task.ts similarity index 100% rename from src/domain/use-cases/task/delete-tasks.ts rename to src/domain/use-cases/task/delete-task.ts diff --git a/src/infra/files/fs-wrapper.ts b/src/infra/files/fs-wrapper.ts new file mode 100644 index 0000000..9d512fe --- /dev/null +++ b/src/infra/files/fs-wrapper.ts @@ -0,0 +1,15 @@ +import { ObjectEncodingOptions } from "node:fs"; + +export interface FsWrapper { + readFile(path: string | Buffer | URL, options: ({ + encoding: BufferEncoding; + flag?: string | number | undefined; + })): Promise + + writeFile(file: string | Buffer | URL, data: string | NodeJS.ArrayBufferView | Iterable | AsyncIterable, options?: (ObjectEncodingOptions & { + mode?: string | number | undefined; + flag?: string | number | undefined; + }) | BufferEncoding | null): Promise + appendFile(path: string | Buffer | URL, data: string | Uint8Array, options?: (ObjectEncodingOptions) | BufferEncoding | null): Promise + +} \ No newline at end of file diff --git a/src/infra/files/fs.ts b/src/infra/files/fs.ts new file mode 100644 index 0000000..f1cc3b4 --- /dev/null +++ b/src/infra/files/fs.ts @@ -0,0 +1,19 @@ +import { FsWrapper } from "./fs-wrapper" +import fs from 'node:fs/promises'; +import { ObjectEncodingOptions } from "fs"; + +// import bcrypt from "bcrypt" +// import { v4 as uuidv4 } from 'uuid'; + +export class FsAdapter implements FsWrapper {//implements readFile, writeFile + async readFile(path: string | Buffer | URL, options: { encoding: BufferEncoding; flag?: string | number | undefined; }): Promise { + return await fs.readFile(path, options) + } + async writeFile(file: string | Buffer | URL, data: string | NodeJS.ArrayBufferView | Iterable | AsyncIterable, options?: ObjectEncodingOptions & { mode?: string | number | undefined; flag?: string | number | undefined; } | BufferEncoding | null): Promise { + return await fs.writeFile(file, data, options) + } + async appendFile(path: string | Buffer | URL, data: string | Uint8Array, options?: ObjectEncodingOptions | BufferEncoding | null): Promise { + return await fs.appendFile(path, data, options) + } +} + diff --git a/src/main.ts b/src/main.ts index 47451c1..1124843 100644 --- a/src/main.ts +++ b/src/main.ts @@ -9,7 +9,6 @@ import UserRouter from './presentation/routers/user-router' import AuthRouter from './presentation/routers/auth-router' import InstrumentModelRouter from './presentation/routers/instrument_model-router' import ProjectRouter from './presentation/routers/project-router' -//import SampleRouter from './presentation/routers/sample-router' import { SearchUsers } from './domain/use-cases/user/search-users' import { CreateUser } from './domain/use-cases/user/create-user' @@ -28,6 +27,11 @@ import { SearchProject } from './domain/use-cases/project/search-projects' import { GetOneInstrumentModel } from './domain/use-cases/instrument_model/get-one-instrument_model' import { SearchInstrumentModels } from './domain/use-cases/instrument_model/search-instrument_model' import { ListImportableSamples } from './domain/use-cases/sample/list-importable-samples' +import { ImportSamples } from './domain/use-cases/sample/import-samples' +import { DeleteTask } from './domain/use-cases/task/delete-task' +import { SearchTask } from './domain/use-cases/task/search-tasks' +import { GetOneTask } from './domain/use-cases/task/get-one-task' +import { GetLogFileTask } from './domain/use-cases/task/get-log-file-task' import { UserRepositoryImpl } from './domain/repositories/user-repository' import { AuthRepositoryImpl } from './domain/repositories/auth-repository' @@ -36,21 +40,25 @@ import { InstrumentModelRepositoryImpl } from './domain/repositories/instrument_ import { ProjectRepositoryImpl } from './domain/repositories/project-repository' import { PrivilegeRepositoryImpl } from './domain/repositories/privilege-repository' import { SampleRepositoryImpl } from './domain/repositories/sample-repository' +import { TaskRepositoryImpl } from './domain/repositories/task-repository' import { SQLiteUserDataSource } from './data/data-sources/sqlite/sqlite-user-data-source' import { SQLiteInstrumentModelDataSource } from './data/data-sources/sqlite/sqlite-instrument_model-data-source' import { SQLiteProjectDataSource } from './data/data-sources/sqlite/sqlite-project-data-source' import { SQLitePrivilegeDataSource } from './data/data-sources/sqlite/sqlite-privilege-data-source' +import { SQLiteTaskDataSource } from './data/data-sources/sqlite/sqlite-task-data-source' import sqlite3 from 'sqlite3' import { BcryptAdapter } from './infra/cryptography/bcript' import { JwtAdapter } from './infra/auth/jsonwebtoken' import { NodemailerAdapter } from './infra/mailer/nodemailer' import { CountriesAdapter } from './infra/countries/country' +import { FsAdapter } from './infra/files/fs' import 'dotenv/config' import path from 'path' +import TaskRouter from './presentation/routers/tasks-router' sqlite3.verbose() @@ -63,6 +71,9 @@ const config = { PORT_PUBLIC: parseInt(process.env.PORT_PUBLIC as string, 10), BASE_URL_PUBLIC: process.env.BASE_URL_PUBLIC || '', + DATA_STORAGE_FOLDER: process.env.DATA_STORAGE_FOLDER || '', + DATA_STORAGE_FS_STORAGE: process.env.DATA_STORAGE_FS_STORAGE || '', + ACCESS_TOKEN_SECRET: process.env.ACCESS_TOKEN_SECRET || '', REFRESH_TOKEN_SECRET: process.env.REFRESH_TOKEN_SECRET || '', VALIDATION_TOKEN_SECRET: process.env.VALIDATION_TOKEN_SECRET || '', @@ -100,11 +111,13 @@ async function getSQLiteDS() { const jwtAdapter = new JwtAdapter() const mailerAdapter = new NodemailerAdapter((config.BASE_URL_PUBLIC + config.PORT_PUBLIC), config.MAIL_SENDER, config.NODE_ENV) const countriesAdapter = new CountriesAdapter() + const fsAdapter = new FsAdapter() const user_dataSource = new SQLiteUserDataSource(db) const instrument_model_dataSource = new SQLiteInstrumentModelDataSource(db) const project_dataSource = new SQLiteProjectDataSource(db) const privilege_dataSource = new SQLitePrivilegeDataSource(db) + const task_datasource = new SQLiteTaskDataSource(db) const transporter = await mailerAdapter.createTransport({ host: config.MAIL_HOST, @@ -123,6 +136,7 @@ async function getSQLiteDS() { const project_repo = new ProjectRepositoryImpl(project_dataSource) const privilege_repo = new PrivilegeRepositoryImpl(privilege_dataSource) const sample_repo = new SampleRepositoryImpl() + const task_repo = new TaskRepositoryImpl(task_datasource, fsAdapter, config.DATA_STORAGE_FOLDER) const userMiddleWare = UserRouter( @@ -154,19 +168,24 @@ async function getSQLiteDS() { new DeleteProject(user_repo, project_repo, privilege_repo), new UpdateProject(user_repo, project_repo, instrument_model_repo, privilege_repo), new SearchProject(user_repo, project_repo, search_repo, instrument_model_repo, privilege_repo), - new ListImportableSamples(sample_repo, user_repo, privilege_repo, project_repo) + new ListImportableSamples(sample_repo, user_repo, privilege_repo, project_repo), + new ImportSamples(sample_repo, user_repo, privilege_repo, project_repo, task_repo, config.DATA_STORAGE_FS_STORAGE) ) - // const sampleMiddleWare = SampleRouter( - // new MiddlewareAuthCookie(jwtAdapter, config.ACCESS_TOKEN_SECRET, config.REFRESH_TOKEN_SECRET), - // - // ) + const taskMiddleWare = TaskRouter( + new MiddlewareAuthCookie(jwtAdapter, config.ACCESS_TOKEN_SECRET, config.REFRESH_TOKEN_SECRET), + new DeleteTask(user_repo, task_repo, privilege_repo), + new GetOneTask(task_repo, user_repo, privilege_repo), + new GetLogFileTask(task_repo, user_repo, privilege_repo), + new SearchTask(user_repo, task_repo, search_repo, project_repo, privilege_repo) + ) server.use("/users", userMiddleWare) server.use("/auth", authMiddleWare) server.use("/instrument_models", instrumentModelMiddleWare) - //server.use("/projects/:project_id/samples", sampleMiddleWare) server.use("/projects", projectMiddleWare) + server.use("/tasks", taskMiddleWare) + server.listen(config.PORT_LOCAL, () => console.log("Running on ", config.BASE_URL_LOCAL, config.PORT_LOCAL)) diff --git a/src/presentation/routers/project-router.ts b/src/presentation/routers/project-router.ts index d53c14c..11e4f09 100644 --- a/src/presentation/routers/project-router.ts +++ b/src/presentation/routers/project-router.ts @@ -7,6 +7,7 @@ import { IMiddlewareProjectValidation } from '../interfaces/middleware/project-v import { CreateProjectUseCase } from '../../domain/interfaces/use-cases/project/create-project' import { DeleteProjectUseCase } from '../../domain/interfaces/use-cases/project/delete-project' import { UpdateProjectUseCase } from '../../domain/interfaces/use-cases/project/update-project' +import { ImportSamplesUseCase } from '../../domain/interfaces/use-cases/sample/import-samples' import { CustomRequest } from '../../domain/entities/auth' import { SearchProjectsUseCase } from '../../domain/interfaces/use-cases/project/search-project' @@ -20,6 +21,7 @@ export default function ProjectRouter( updateProjectUseCase: UpdateProjectUseCase, searchProjectUseCase: SearchProjectsUseCase, listImportableSamples: ListImportableSamplesUseCase, + importSamples: ImportSamplesUseCase, ) { const router = express.Router() @@ -139,20 +141,20 @@ export default function ProjectRouter( } }) - // // Pagined and sorted list of filtered task - // router.post('/:project_id/samples/import', middlewareAuth.auth,/*middlewareSampleValidation.rulesImport,*/ async (req: Request, res: Response) => { - // try { - // const tasks = await importSamples.execute((req as CustomRequest).token, { ...req.body, project_id: req.params.project_id }); - // res.status(200).send(tasks) - // } catch (err) { - // // console.log(err) - // // if (err.message === "User cannot be used") res.status(403).send({ errors: [err.message] }) - // // else if (err.message === "Task type label not found") res.status(404).send({ errors: [err.message] }) - // // else if (err.message === "Task status label not found") res.status(404).send({ errors: [err.message] }) - // // else res.status(500).send({ errors: ["Cannot search tasks"] }) - // res.status(500).send({ errors: ["Cannot search tasks"] }) - // } - // }) + // Pagined and sorted list of filtered task + router.post('/:project_id/samples/import', middlewareAuth.auth,/*middlewareSampleValidation.rulesImport,*/ async (req: Request, res: Response) => { + try { + const tasks = await importSamples.execute((req as CustomRequest).token, req.params.project_id as any, { ...req.body }.samples); + res.status(200).send(tasks) + } catch (err) { + // console.log(err) + // if (err.message === "User cannot be used") res.status(403).send({ errors: [err.message] }) + // else if (err.message === "Task type label not found") res.status(404).send({ errors: [err.message] }) + // else if (err.message === "Task status label not found") res.status(404).send({ errors: [err.message] }) + // else res.status(500).send({ errors: ["Cannot search tasks"] }) + res.status(500).send({ errors: ["Cannot search tasks"] }) + } + }) return router } \ No newline at end of file diff --git a/src/presentation/routers/tasks-router.ts b/src/presentation/routers/tasks-router.ts index 4cc9d56..cae06e8 100644 --- a/src/presentation/routers/tasks-router.ts +++ b/src/presentation/routers/tasks-router.ts @@ -52,7 +52,7 @@ export default function TaskRouter( // Get one task router.get('/:task_id/', middlewareAuth.auth,/*middlewareTaskValidation.rulesGetTasks,*/ async (req: Request, res: Response) => { try { - const task = await getOneTaskUseCase.execute((req as CustomRequest).token, { ...req.body, task_id: req.params.task_id }) + const task = await getOneTaskUseCase.execute((req as CustomRequest).token, req.params.task_id as any); res.status(200).send(task) } catch (err) { console.log(err)