DEV - Backup L0b : OK

src/data/data-sources/sqlite/sqlite-task-data-source.ts

@@ -78,9 +78,11 @@ export class SQLiteTaskDataSource implements TaskDataSource {
                 INSERT OR IGNORE INTO task_type (task_type_label) 
                 VALUES 
                 ('EXPORT'),
+                ('EXPORT_BACKUP'),
                 ('DELETE'),
                 ('UPDATE'),
                 ('IMPORT'),
+                ('IMPORT_BACKUP'),
                 ('IMPORT_CTD'),
                 ('IMPORT_ECO_TAXA');
                 `;
@@ -330,7 +332,7 @@ export class SQLiteTaskDataSource implements TaskDataSource {
         // generate sql and params
         for (const [key, value] of Object.entries(task)) {
             params.push(value)
-            placeholders = placeholders + key + "=(?) AND "
+            placeholders = placeholders + "task." + key + "=(?) AND "
         }
         // remove last AND
         placeholders = placeholders.slice(0, -4);

src/domain/entities/task.ts

@@ -12,6 +12,8 @@ export enum TaskType {
     Delete = "DELETE",
     Update = "UPDATE",
     Import = "IMPORT",
+    Import_Backup = "IMPORT_BACKUP",
+    Export_Backup = "EXPORT_BACKUP",
     Import_CTD = "IMPORT_CTD",
     Import_EcoTaxa = "IMPORT_ECO_TAXA",
 }
@@ -99,8 +101,8 @@ export interface TaskResponseModel {
 
 export interface PrivateTaskRequestModel {
     task_id?: number;
-    task_type_id?: TaskType;
-    task_status_id?: TasksStatus;
+    task_type_id?: number;
+    task_status_id?: number;
     task_owner_id?: number;
     task_project_id?: number;
     task_params?: object;

src/domain/interfaces/repositories/project-repository.ts

@@ -13,4 +13,6 @@ export interface ProjectRepository {
     computeDefaultDepthOffset(instrument_model: string): number | undefined;
     deleteProject(project: ProjectRequestModel): Promise<number>;
     standardGetProjects(options: PreparedSearchOptions): Promise<SearchResult<ProjectResponseModel>>;
+    ensureFolderStructureForBackup(root_folder_path: string): Promise<void>;
+    copyL0bToProjectFolder(source_folder: string, dest_folder: string, skip_already_imported: boolean): Promise<void>;
 }

src/domain/interfaces/use-cases/project/backup-project.ts

@@ -0,0 +1,6 @@
+import { TaskResponseModel } from "../../../entities/task";
+import { UserUpdateModel } from "../../../entities/user";
+
+export interface BackupProjectUseCase {
+    execute(current_user: UserUpdateModel, project_id: number, skip_already_imported: boolean): Promise<TaskResponseModel>;
+}

src/domain/repositories/project-repository.ts

@@ -6,14 +6,20 @@ import { ProjectRequestCreationModel, ProjectRequestModel, ProjectUpdateModel, P
 import { PreparedSearchOptions, SearchResult } from "../entities/search";
 import { ProjectRepository } from "../interfaces/repositories/project-repository";
 
-import { promises as fs } from 'fs';
+import * as fsPromises from 'fs/promises'; // For promise-based file operations//import fs from 'fs'; // Correct import for `createReadStream`
+import * as fs from 'fs'; // For createWriteStream
+
+import archiver from 'archiver';
+
+import path from "path";
 
 export class ProjectRepositoryImpl implements ProjectRepository {
     projectDataSource: ProjectDataSource
 
     // TODO move to a search repository
     order_by_allow_params: string[] = ["asc", "desc"]
     filter_operator_allow_params: string[] = ["=", ">", "<", ">=", "<=", "<>", "IN", "LIKE"]
+    base_folder = path.join(__dirname, '..', '..', '..');
 
     constructor(projectDataSource: ProjectDataSource) {
         this.projectDataSource = projectDataSource
@@ -186,13 +192,160 @@ export class ProjectRepositoryImpl implements ProjectRepository {
     async createProjectRootFolder(root_folder_path: string): Promise<void> {
         try {
             // Check if the folder exists
-            await fs.access(root_folder_path);
+            await fsPromises.access(root_folder_path);
             // If it exists, remove it recursively
-            await fs.rm(root_folder_path, { recursive: true, force: true });
+            await fsPromises.rm(root_folder_path, { recursive: true, force: true });
         } catch (error) {
             // Folder does not exist; no need to delete anything
         }
         // Create the root folder
-        await fs.mkdir(root_folder_path, { recursive: true });
+        await fsPromises.mkdir(root_folder_path, { recursive: true });
+    }
+
+    async ensureFolderStructureForBackup(root_folder_path: string): Promise<void> {
+        // Ensure /raw, /meta, /config EXISTS
+        const foldersTocheck = ['raw', 'meta', 'config'];
+
+        for (const folder of foldersTocheck) {
+            const folderPath = path.join(this.base_folder, root_folder_path, folder);
+            try {
+                await fsPromises.access(folderPath);
+            } catch (error) {
+                throw new Error(`Folder does not exist at path: ${folderPath}`);
+            }
+        }
     }
+
+    async copyL0bToProjectFolder(source_folder: string, dest_folder: string, skip_already_imported: boolean): Promise<void> {
+        // Create destination folder if does not exist
+        await fsPromises.mkdir(path.join(this.base_folder, dest_folder), { recursive: true });
+
+        // Copy meta/*, and config/*.
+        await this.copy_metadata(this.base_folder, source_folder, dest_folder);
+
+        // Copy L0b files
+        if (skip_already_imported === true) {
+            await this.copyNewL0bFolders(this.base_folder, source_folder, dest_folder);
+        } else {
+            await this.copyAllL0bFolders(this.base_folder, source_folder, dest_folder);
+        }
+    }
+
+    async copy_metadata(base_folder: string, source_folder: string, dest_folder: string): Promise<void> {
+        const foldersToCopy = [
+            { source: 'meta', dest: 'meta' },
+            { source: 'config', dest: 'config' }
+        ];
+
+        for (const folder of foldersToCopy) {
+            const sourcePath = path.join(base_folder, source_folder, folder.source);
+            const destPath = path.join(base_folder, dest_folder, folder.dest);
+            const oldDestPath = path.join(base_folder, dest_folder, `old_${folder.dest}`);
+
+            // Ensure the destination folder exists
+            await fsPromises.mkdir(destPath, { recursive: true });
+
+            // Rename dest folder to old_{folder.dest}
+            await fsPromises.rename(destPath, oldDestPath);
+            try {
+                // Copy source folder to dest folder using cp
+                await fsPromises.cp(sourcePath, destPath, { recursive: true });
+            } catch (error) {
+                // If an error occurs, restore the old_{folder.dest}
+                await fsPromises.rename(oldDestPath, destPath);
+                throw error;
+            }
+
+            // If everything is ok, remove old_{folder.dest}
+            await fsPromises.rm(oldDestPath, { recursive: true, force: true });
+        }
+    }
+
+
+    async copyAllL0bFolders(base_folder: string, source_folder: string, dest_folder: string): Promise<void> {
+        const sourcePath = path.join(base_folder, source_folder, 'raw');
+        const destPath = path.join(base_folder, dest_folder, 'raw');
+
+        // Ensure the destination folder exists
+        await fsPromises.mkdir(destPath, { recursive: true });
+
+        // Read all subfolders in the source folder
+        const subfolders = await fsPromises.readdir(sourcePath, { withFileTypes: true });
+
+        for (const entry of subfolders) {
+            if (entry.isDirectory()) {
+                const sourceSubfolder = path.join(sourcePath, entry.name);
+                const destZipFile = path.join(destPath, `${entry.name}.zip`);
+
+                // Zip the folder and write to the destination
+                await this.zipFolder(sourceSubfolder, destZipFile);
+            }
+        }
+    }
+
+    async zipFolder(sourceFolder: string, destZipFile: string): Promise<void> {
+        // Create a file output stream for the destination zip file
+        const output = fs.createWriteStream(destZipFile);
+
+        // Create a new archiver instance to create a zip file
+        const archive = archiver('zip', {
+            zlib: { level: 9 }  // Maximum compression
+        });
+
+        // Pipe the archiver output to the file stream
+        output.on('close', () => {
+            //console.log(`Archive ${destZipFile} has been created, ${archive.pointer()} total bytes.`);
+        });
+
+        archive.on('error', (err) => {
+            throw err;
+        });
+
+        archive.pipe(output);
+
+        // Append all files from the source folder to the archive
+        archive.directory(sourceFolder, false);  // Second argument 'false' keeps directory structure intact
+
+        // Finalize the archive (this will compress the data)
+        await archive.finalize();
+    }
+
+    async copyNewL0bFolders(base_folder: string, source_folder: string, dest_folder: string): Promise<void> {
+        const sourcePath = path.join(base_folder, source_folder, 'raw');
+        const destPath = path.join(base_folder, dest_folder, 'raw');
+
+        // Ensure the destination folder exists
+        await fsPromises.mkdir(destPath, { recursive: true });
+
+        // Read all subfolders in the source folder
+        const subfolders = await fsPromises.readdir(sourcePath, { withFileTypes: true });
+
+        for (const entry of subfolders) {
+            if (entry.isDirectory()) {
+                const sourceSubfolder = path.join(sourcePath, entry.name);
+                const destSubfolder = path.join(destPath, `${entry.name}.zip`);
+
+                // Check if the subfolder is already copied (check if the zip file exists)
+                const zipExists = await this.checkFileExists(destSubfolder);
+                if (!zipExists) {
+                    // If not, copy and zip the folder
+                    //console.log(`Copying new folder: ${entry.name}`);
+                    await this.zipFolder(sourceSubfolder, destSubfolder);
+                }
+                // else {
+                //     console.log(`Skipping already existing folder: ${entry.name}`);
+                // }
+            }
+        }
+    }
+
+    async checkFileExists(filePath: string): Promise<boolean> {
+        try {
+            await fsPromises.access(filePath);
+            return true; // File exists
+        } catch {
+            return false; // File does not exist
+        }
+    }
+
 }

src/domain/repositories/sample-repository.ts

@@ -30,6 +30,7 @@ export class SampleRepositoryImpl implements SampleRepository {
     // TODO move to a search repository
     order_by_allow_params: string[] = ["asc", "desc"]
     filter_operator_allow_params: string[] = ["=", ">", "<", ">=", "<=", "<>", "IN", "LIKE"]
+    base_folder = path.join(__dirname, '..', '..', '..');
 
     constructor(sampleDataSource: SampleDataSource, DATA_STORAGE_FS_STORAGE: string) {
         this.sampleDataSource = sampleDataSource
@@ -894,18 +895,17 @@ export class SampleRepositoryImpl implements SampleRepository {
         }
     }
     async UVP5copySamplesToImportFolder(source_folder: string, dest_folder: string, samples_names_to_import: string[]): Promise<void> {
-        const base_folder = path.join(__dirname, '..', '..', '..');
 
         // Ensure that none of the samples folder already exists
-        await this.ensureSampleFolderDoNotExists(samples_names_to_import, path.join(base_folder, dest_folder));
+        await this.ensureSampleFolderDoNotExists(samples_names_to_import, path.join(this.base_folder, dest_folder));
 
         // Create destination folder
-        await fsPromises.mkdir(path.join(base_folder, dest_folder), { recursive: true });
+        await fsPromises.mkdir(path.join(this.base_folder, dest_folder), { recursive: true });
 
         // Iterate over each sample name, create the sample folder, copy files, and zip the folder
         for (const sample of samples_names_to_import) {
-            const sourcePath = path.join(base_folder, source_folder);
-            const destPath = path.join(base_folder, dest_folder, sample);
+            const sourcePath = path.join(this.base_folder, source_folder);
+            const destPath = path.join(this.base_folder, dest_folder, sample);
 
             // Ensure the destination sample folder exists
             await fsPromises.mkdir(destPath, { recursive: true });
@@ -935,7 +935,7 @@ export class SampleRepositoryImpl implements SampleRepository {
             }
 
             // Zip the sample folder
-            const zipFilePath = path.join(base_folder, dest_folder, `${sample}.zip`);
+            const zipFilePath = path.join(this.base_folder, dest_folder, `${sample}.zip`);
             try {
                 await this.zipFolder(destPath, zipFilePath);
 
@@ -962,18 +962,16 @@ export class SampleRepositoryImpl implements SampleRepository {
     }
 
     async UVP6copySamplesToImportFolder(source_folder: string, dest_folder: string, samples_names_to_import: string[]): Promise<void> {
-
-        const base_folder = path.join(__dirname, '..', '..', '..');
         // Ensure that non of the samples folder already exists
-        await this.ensureSampleFolderDoNotExists(samples_names_to_import, path.join(base_folder, dest_folder));
+        await this.ensureSampleFolderDoNotExists(samples_names_to_import, path.join(this.base_folder, dest_folder));
 
         // Ensure destination folder exists
-        await fsPromises.mkdir(path.join(base_folder, dest_folder), { recursive: true });
+        await fsPromises.mkdir(path.join(this.base_folder, dest_folder), { recursive: true });
 
         // Iterate over each sample name and copy .zip files only
         for (const sample of samples_names_to_import) {
-            const sourcePath = path.join(base_folder, source_folder, sample);
-            const destPath = path.join(base_folder, dest_folder, sample);
+            const sourcePath = path.join(this.base_folder, source_folder, sample);
+            const destPath = path.join(this.base_folder, dest_folder, sample);
 
             // Check if the sample directory exists and list files
             const files = await fsPromises.readdir(sourcePath);
@@ -993,12 +991,10 @@ export class SampleRepositoryImpl implements SampleRepository {
     }
 
     async deleteSamplesFromImportFolder(dest_folder: string, samples_names_to_import: string[]): Promise<void> {
-        const base_folder = path.join(__dirname, '..', '..', '..');
-
         for (const sample of samples_names_to_import) {
             // Construct paths for the .zip file and the folder
-            const zipPath = path.join(base_folder, dest_folder, `${sample}.zip`);
-            const folderPath = path.join(base_folder, dest_folder, sample);
+            const zipPath = path.join(this.base_folder, dest_folder, `${sample}.zip`);
+            const folderPath = path.join(this.base_folder, dest_folder, sample);
 
             // Delete the .zip file
             await fsPromises.rm(zipPath, { force: true });

src/domain/repositories/task-repository.ts

@@ -20,6 +20,8 @@ export class TaskRepositoryImpl implements TaskRepository {
     // TODO move to a search repository
     order_by_allow_params: string[] = ["asc", "desc"]
     filter_operator_allow_params: string[] = ["=", ">", "<", ">=", "<=", "<>", "IN", "LIKE"]
+    base_folder = path.join(__dirname, '..', '..', '..');
+
 
     constructor(taskDataSource: TaskDataSource, fs: FsWrapper, DATA_STORAGE_FOLDER: string) {
         this.taskDataSource = taskDataSource
@@ -62,7 +64,7 @@ export class TaskRepositoryImpl implements TaskRepository {
         const result = await this.taskDataSource.create(private_task)
 
         //create log file based on created task_id
-        const log_file_path = path.join(__dirname, '..', '..', '..', this.DATA_STORAGE_FOLDER, "tasks_log", `task_${result}.log`)
+        const log_file_path = path.join(this.base_folder, this.DATA_STORAGE_FOLDER, "tasks_log", `task_${result}.log`)
 
         //  create log file
         try {