From d4df90f1067a1b8ec3da6e6f41a7d9e212690f55 Mon Sep 17 00:00:00 2001 From: chenyufeifei Date: Sat, 4 Nov 2023 20:50:53 +0800 Subject: [PATCH 1/5] feat: add dockerfile Signed-off-by: chenyufeifei --- CMakeLists.txt | 9 +++++--- Dockerfile | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 Dockerfile diff --git a/CMakeLists.txt b/CMakeLists.txt index 686f863..405335a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -101,16 +101,19 @@ target_link_libraries(triton-dragonfly-repoagent PRIVATE re2::re2) # if(${TRITON_ENABLE_S3}) find_package(ZLIB REQUIRED) - find_package(AWSSDK REQUIRED COMPONENTS s3) + find_package(AWSSDK REQUIRED COMPONENTS core s3) message(STATUS "Using aws-sdk-cpp ${AWSSDK_VERSION}") target_include_directories( triton-dragonfly-repoagent - PRIVATE $ + PRIVATE + $ + $ ) + target_link_libraries( triton-dragonfly-repoagent PRIVATE - aws-cpp-sdk-s3 + aws-cpp-sdk-s3 aws-cpp-sdk-core ) endif() # TRITON_ENABLE_S3 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8a2e79f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,56 @@ +# Use a base image that includes development essentials and git +FROM ubuntu:latest AS builder + +# Install build tools, git, and other dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + git \ + curl \ + zip \ + unzip \ + tar \ + pkg-config \ + python3 + +# Install vcpkg +RUN git clone https://github.com/Microsoft/vcpkg.git /vcpkg \ + && /vcpkg/bootstrap-vcpkg.sh + +# Install the required dependencies using vcpkg +RUN /vcpkg/vcpkg install re2 \ + && /vcpkg/vcpkg install aws-sdk-cpp \ + && /vcpkg/vcpkg install google-cloud-cpp[storage] \ + && /vcpkg/vcpkg install azure-storage-blobs-cpp \ + && /vcpkg/vcpkg install rapidjson + +RUN mkdir -p /dragonfly-repository-agent/build + +# Assuming the source code is located in a directory called 'source' +# You can COPY this in or clone from a repository as needed +COPY ./src /dragonfly-repository-agent/src +COPY ./cmake /dragonfly-repository-agent/cmake +COPY ./CMakeLists.txt /dragonfly-repository-agent/CMakeLists.txt + +# Set the working directory +WORKDIR /dragonfly-repository-agent/build + +# Set the environment variable for the vcpkg toolchain file +ENV VCPKG_TOOLCHAIN_FILE=/vcpkg/scripts/buildsystems/vcpkg.cmake + +# Build the repository agent +RUN cmake -DCMAKE_TOOLCHAIN_FILE=${VCPKG_TOOLCHAIN_FILE} \ + -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install \ + -DTRITON_ENABLE_GCS=true \ + -DTRITON_ENABLE_AZURE_STORAGE=true \ + -DTRITON_ENABLE_S3=true \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ + .. + +RUN make install + +FROM nvcr.io/nvidia/tritonserver:23.08-py3 + +RUN mkdir /opt/tritonserver/repoagents/dragonfly + +COPY --from=builder /dragonfly-repository-agent/build/libtritonrepoagent_dragonfly.so /opt/tritonserver/repoagents/dragonfly/libtritonrepoagent_dragonfly.so From 80cae069e2ee29dd6d9375de132994dcdb04b72a Mon Sep 17 00:00:00 2001 From: chenyufeifei Date: Sat, 4 Nov 2023 21:03:54 +0800 Subject: [PATCH 2/5] feat: add workflow Signed-off-by: chenyufeifei --- .github/workflows/build.yml | 72 +++++++++++++++++++++++++++++++++++++ .github/workflows/lint.yml | 30 ++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..5121e00 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,72 @@ +name: Build + +on: + push: + branches: + - main + tags: + - v* + +jobs: + push_image_to_registry: + name: Push Image + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Get Version + id: get_version + run: | + VERSION=${GITHUB_REF#refs/tags/} + if [[ ${GITHUB_REF} == "refs/heads/main" ]]; then + VERSION=latest + fi + echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT + + - name: Get Git Revision + id: vars + shell: bash + run: | + echo "git_revision=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: PrepareReg Names + run: | + echo IMAGE_REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login Docker Hub + uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push to Registry + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64 + file: Dockerfile + labels: |- + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + build-args: | + GITVERSION=git-${{ steps.vars.outputs.git_revision }} + VERSION=${{ steps.get_version.outputs.VERSION }} + tags: | + dragonflyoss/tritonserver-dragonfly:${{ steps.get_version.outputs.VERSION }} + ghcr.io/${{ env.IMAGE_REPOSITORY }}/tritonserver-dragonfly:${{ steps.get_version.outputs.VERSION }} + push: + true \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..a25bf0a --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,30 @@ +name: Lint + +on: + push: + branches: [main, release-*] + pull_request: + branches: [main, release-*] + +jobs: + lint: + name: CPP Lint + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v2 + + - name: Install clang-format + run: sudo apt-get install -y clang-format + + - name: Run clang-format + run: | + find . -iname *.h -o -iname *.cpp | xargs clang-format -i + + - name: Check for changes + run: | + if ! git diff --exit-code; then + echo "Code style issues found" + git diff + exit 1 + fi \ No newline at end of file From 8b3cab6944050b169d265d7c79a24d9958fbab56 Mon Sep 17 00:00:00 2001 From: chenyufeifei Date: Sat, 4 Nov 2023 21:20:58 +0800 Subject: [PATCH 3/5] Format code with clang-format Signed-off-by: chenyufeifei --- src/common_utils.h | 75 ++++++++++--------- src/config.h | 32 ++++---- src/dragonfly.cpp | 34 ++++----- src/filesystem/api.cpp | 98 ++++++++++++------------- src/filesystem/api.h | 26 +++---- src/filesystem/implementations/as.h | 67 +++++++++-------- src/filesystem/implementations/common.h | 31 ++++---- src/filesystem/implementations/gcs.h | 85 ++++++++++----------- src/filesystem/implementations/s3.h | 98 +++++++++++++------------ src/status.h | 16 ++-- 10 files changed, 289 insertions(+), 273 deletions(-) diff --git a/src/common_utils.h b/src/common_utils.h index 2063076..ce1e427 100644 --- a/src/common_utils.h +++ b/src/common_utils.h @@ -1,28 +1,28 @@ /* -* Copyright 2023 The Dragonfly Authors -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. + * Copyright 2023 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #pragma once #include -#include #include +#include #include "config.h" -#include "triton/core/tritonserver.h" #include "curl/curl.h" +#include "triton/core/tritonserver.h" namespace triton::repoagent::dragonfly { @@ -92,14 +92,15 @@ BaseName(const std::string& path) } TRITONSERVER_Error* -DownloadFile (const std::string &url, const std::string &path, DragonflyConfig& config) { +DownloadFile( + const std::string& url, const std::string& path, DragonflyConfig& config) +{ CURL* curl; curl = curl_easy_init(); if (!curl) { return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INTERNAL, - "Failed to initialize CURL."); + TRITONSERVER_ERROR_INTERNAL, "Failed to initialize CURL."); } FILE* fp = fopen(path.c_str(), "wb"); @@ -111,15 +112,18 @@ DownloadFile (const std::string &url, const std::string &path, DragonflyConfig& } - struct curl_slist *headers = NULL; + struct curl_slist* headers = NULL; auto cleanup = [&]() { - if (fp) fclose(fp); - if (headers) curl_slist_free_all(headers); + if (fp) + fclose(fp); + if (headers) + curl_slist_free_all(headers); curl_easy_cleanup(curl); }; - auto write_data = [](void* ptr, size_t size, size_t nmemb, void* stream) -> size_t { + auto write_data = [](void* ptr, size_t size, size_t nmemb, + void* stream) -> size_t { size_t written = fwrite(ptr, size, nmemb, static_cast(stream)); return written; }; @@ -128,29 +132,29 @@ DownloadFile (const std::string &url, const std::string &path, DragonflyConfig& curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data); curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); - for(const auto& header : config.headers) { + for (const auto& header : config.headers) { std::string header_str = header.first + ": " + header.second; headers = curl_slist_append(headers, header_str.c_str()); if (!headers) { cleanup(); return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INTERNAL, - "Failed to append headers."); + TRITONSERVER_ERROR_INTERNAL, "Failed to append headers."); } } if (!config.filter.empty()) { std::ostringstream oss; for (size_t i = 0; i < config.filter.size(); ++i) { - if (i != 0) oss << "&"; + if (i != 0) + oss << "&"; oss << config.filter[i]; } - headers = curl_slist_append(headers, ("X-Dragonfly-Filter: " + oss.str()).c_str()); + headers = curl_slist_append( + headers, ("X-Dragonfly-Filter: " + oss.str()).c_str()); if (!headers) { cleanup(); return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INTERNAL, - "Failed to append filters."); + TRITONSERVER_ERROR_INTERNAL, "Failed to append filters."); } } @@ -162,11 +166,10 @@ DownloadFile (const std::string &url, const std::string &path, DragonflyConfig& CURLcode res = curl_easy_perform(curl); - if(res != CURLE_OK) { + if (res != CURLE_OK) { cleanup(); return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INTERNAL, - curl_easy_strerror(res)); + TRITONSERVER_ERROR_INTERNAL, curl_easy_strerror(res)); } cleanup(); @@ -174,11 +177,11 @@ DownloadFile (const std::string &url, const std::string &path, DragonflyConfig& } TRITONSERVER_Error* -ReadLocalFile(const std::string &path, std::string *contents) { +ReadLocalFile(const std::string& path, std::string* contents) +{ if (!contents) { return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INTERNAL, - "Output string pointer is null."); + TRITONSERVER_ERROR_INTERNAL, "Output string pointer is null."); } std::ifstream in(path, std::ios::in | std::ios::binary); @@ -201,4 +204,4 @@ ReadLocalFile(const std::string &path, std::string *contents) { return nullptr; } -} \ No newline at end of file +} // namespace triton::repoagent::dragonfly \ No newline at end of file diff --git a/src/config.h b/src/config.h index b4510b9..1db06bc 100644 --- a/src/config.h +++ b/src/config.h @@ -1,22 +1,22 @@ /* -* Copyright 2023 The Dragonfly Authors -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. + * Copyright 2023 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #pragma once -#include #include +#include #include #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* @@ -25,7 +25,7 @@ #define TRITONJSON_STATUSSUCCESS nullptr #include "triton/common/triton_json.h" -namespace triton::repoagent::dragonfly{ +namespace triton::repoagent::dragonfly { struct DragonflyConfig { std::string proxy; @@ -63,4 +63,4 @@ DragonflyConfig::DragonflyConfig(triton::common::TritonJson::Value& config) } } } -} \ No newline at end of file +} // namespace triton::repoagent::dragonfly \ No newline at end of file diff --git a/src/dragonfly.cpp b/src/dragonfly.cpp index 40eee0b..c2f93bf 100644 --- a/src/dragonfly.cpp +++ b/src/dragonfly.cpp @@ -1,17 +1,17 @@ /* -* Copyright 2023 The Dragonfly Authors -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. + * Copyright 2023 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #include @@ -32,7 +32,6 @@ TRITONREPOAGENT_ModelAction( { switch (action_type) { case TRITONREPOAGENT_ACTION_LOAD: { - const char* location_cstr; TRITONREPOAGENT_ArtifactType artifact_type; RETURN_IF_ERROR(TRITONREPOAGENT_ModelRepositoryLocation( @@ -60,7 +59,8 @@ TRITONREPOAGENT_ModelAction( const std::string temp_dir(temp_dir_cstr); try { - RETURN_IF_ERROR(LocalizePath(config_path, cred_path, location, temp_dir)); + RETURN_IF_ERROR( + LocalizePath(config_path, cred_path, location, temp_dir)); char* l = const_cast(temp_dir.c_str()); RETURN_IF_ERROR(TRITONREPOAGENT_ModelRepositoryUpdate( @@ -77,5 +77,5 @@ TRITONREPOAGENT_ModelAction( } } // extern "C" - -}} \ No newline at end of file +} +} // namespace triton::repoagent::dragonfly \ No newline at end of file diff --git a/src/filesystem/api.cpp b/src/filesystem/api.cpp index 9a16689..9e0dadd 100644 --- a/src/filesystem/api.cpp +++ b/src/filesystem/api.cpp @@ -1,23 +1,24 @@ /* -* Copyright 2023 The Dragonfly Authors -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. + * Copyright 2023 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #include "api.h" -#include "implementations/common.h" -#include "triton/core/tritonserver.h" + #include "common_utils.h" #include "config.h" +#include "implementations/common.h" +#include "triton/core/tritonserver.h" #define TRITON_ENABLE_GCS #define TRITON_ENABLE_S3 @@ -36,7 +37,6 @@ #endif // TRITON_ENABLE_AZURE_STORAGE - #include namespace triton::repoagent::dragonfly { @@ -46,38 +46,34 @@ namespace { class FileSystemManager { public: TRITONSERVER_Error* GetFileSystem( - const std::string& path, - std::shared_ptr& file_system, + const std::string& path, std::shared_ptr& file_system, const std::string& cred_path); // 创建file_system private: - template + template TRITONSERVER_Error* GetFileSystem( - const std::string& path, - CacheType& cache, - std::shared_ptr& file_system, - const std::string& cred_path); + const std::string& path, CacheType& cache, + std::shared_ptr& file_system, const std::string& cred_path); TRITONSERVER_Error* LoadCredentials(const std::string& cred_path); - template + template static void LoadCredential( - triton::common::TritonJson::Value &creds_json, - const char *fs_type, - CacheType &cache); + triton::common::TritonJson::Value& creds_json, const char* fs_type, + CacheType& cache); template - static void SortCache(std::vector>>& cache); + static void SortCache( + std::vector>>& + cache); template static TRITONSERVER_Error* GetLongestMatchingNameIndex( const std::vector>>& cache, - const std::string& path, - size_t& idx); + const std::string& path, size_t& idx); #ifdef TRITON_ENABLE_GCS std::vector< @@ -98,7 +94,9 @@ class FileSystemManager { TRITONSERVER_Error* FileSystemManager::GetFileSystem( - const std::string& path, std::shared_ptr& file_system, const std::string& cred_path) { + const std::string& path, std::shared_ptr& file_system, + const std::string& cred_path) +{ // Check if this is a GCS path (gs://$BUCKET_NAME) if (!path.empty() && !path.rfind("gs://", 0)) { #ifndef TRITON_ENABLE_GCS @@ -108,7 +106,8 @@ FileSystemManager::GetFileSystem( "-DTRITON_ENABLE_GCS=ON."); #else return GetFileSystem< - std::vector>>, + std::vector>>, GCSCredential, GCSFileSystem>(path, gs_cache_, file_system, cred_path); #endif // TRITON_ENABLE_GCS } @@ -144,8 +143,7 @@ FileSystemManager::GetFileSystem( } return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_UNSUPPORTED, - "filesystem type error"); + TRITONSERVER_ERROR_UNSUPPORTED, "filesystem type error"); } TRITONSERVER_Error* @@ -168,15 +166,15 @@ FileSystemManager::LoadCredentials(const std::string& cred_path) #ifdef TRITON_ENABLE_S3 // load S3 credentials LoadCredential< - std::vector>>, + std::vector< + std::tuple>>, S3Credential, S3FileSystem>(creds_json, "s3", s3_cache_); #endif // TRITON_ENABLE_S3 #ifdef TRITON_ENABLE_AZURE_STORAGE // load AS credentials LoadCredential< - std::vector>>, + std::vector< + std::tuple>>, ASCredential, ASFileSystem>(creds_json, "as", as_cache_); #endif // TRITON_ENABLE_AZURE_STORAGE return nullptr; @@ -212,8 +210,7 @@ template TRITONSERVER_Error* FileSystemManager::GetFileSystem( const std::string& path, CacheType& cache, - std::shared_ptr& file_system, - const std::string& cred_path) + std::shared_ptr& file_system, const std::string& cred_path) { RETURN_IF_ERROR(LoadCredentials(cred_path)); @@ -221,7 +218,8 @@ FileSystemManager::GetFileSystem( RETURN_IF_ERROR(GetLongestMatchingNameIndex(cache, path, idx)); CredentialType cred = std::get<1>(cache[idx]); - std::shared_ptr fs = std::make_shared(path, cred); + std::shared_ptr fs = + std::make_shared(path, cred); RETURN_IF_ERROR(fs->CheckClient(path)); file_system = fs; return nullptr; @@ -229,8 +227,9 @@ FileSystemManager::GetFileSystem( template void -FileSystemManager::SortCache(std::vector>>& cache) +FileSystemManager::SortCache( + std::vector>>& cache) { std::sort( cache.begin(), cache.end(), @@ -261,11 +260,12 @@ FileSystemManager::GetLongestMatchingNameIndex( } FileSystemManager fsm_; -} +} // namespace TRITONSERVER_Error* -LocalizePath(const std::string& config_path, const std::string& cred_path, - const std::string& location, const std::string& temp_dir) +LocalizePath( + const std::string& config_path, const std::string& cred_path, + const std::string& location, const std::string& temp_dir) { std::shared_ptr fs; RETURN_IF_ERROR(fsm_.GetFileSystem(location, fs, cred_path)); @@ -279,4 +279,4 @@ LocalizePath(const std::string& config_path, const std::string& cred_path, return fs->LocalizePath(location, temp_dir, config); } -} \ No newline at end of file +} // namespace triton::repoagent::dragonfly \ No newline at end of file diff --git a/src/filesystem/api.h b/src/filesystem/api.h index 63d6d51..f0d31da 100644 --- a/src/filesystem/api.h +++ b/src/filesystem/api.h @@ -1,17 +1,17 @@ /* -* Copyright 2023 The Dragonfly Authors -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. + * Copyright 2023 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #include diff --git a/src/filesystem/implementations/as.h b/src/filesystem/implementations/as.h index c0cb70d..9ee4d49 100644 --- a/src/filesystem/implementations/as.h +++ b/src/filesystem/implementations/as.h @@ -1,27 +1,26 @@ /* -* Copyright 2023 The Dragonfly Authors -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. + * Copyright 2023 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #pragma once #include "azure/storage/blobs.hpp" #include "azure/storage/common/storage_credential.hpp" -#include "vector" - #include "common.h" #include "common_utils.h" +#include "vector" #undef LOG_INFO #undef LOG_WARNING @@ -56,7 +55,8 @@ class ASFileSystem : public FileSystem { TRITONSERVER_Error* CheckClient(const std::string& path); TRITONSERVER_Error* LocalizePath( - const std::string& location, const std::string& temp_dir, DragonflyConfig& config_path) override; + const std::string& location, const std::string& temp_dir, + DragonflyConfig& config_path) override; TRITONSERVER_Error* FileExists(const std::string& path, bool* exists); @@ -73,8 +73,7 @@ class ASFileSystem : public FileSystem { const std::string& container, const std::string& dir_path, const std::function& blobs, - const std::vector& blob_prefixes)>& - callback); + const std::vector& blob_prefixes)>& callback); TRITONSERVER_Error* DownloadFolder( const std::string& container, const std::string& path, @@ -91,7 +90,8 @@ ASFileSystem::ParsePath( std::string host_name, query; if (!RE2::FullMatch(path, as_regex_, &host_name, container, blob, &query)) { return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INTERNAL, ("Invalid azure storage path: " + path).c_str()); + TRITONSERVER_ERROR_INTERNAL, + ("Invalid azure storage path: " + path).c_str()); } return nullptr; } @@ -196,7 +196,8 @@ ASFileSystem::FileExists(const std::string& path, bool* exists) catch (as::StorageException& ex) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - ("Failed to check if file exists at " + path + ":" + ex.what()).c_str()); + ("Failed to check if file exists at " + path + ":" + ex.what()) + .c_str()); } return nullptr; @@ -209,17 +210,20 @@ ASFileSystem::DownloadFolder( { auto container_client = client_->GetBlobContainerClient(container); auto func = [&](const std::vector& blobs, - const std::vector& blob_prefixes) -> TRITONSERVER_Error* { + const std::vector& blob_prefixes) + -> TRITONSERVER_Error* { for (const auto& blob_item : blobs) { const auto& local_path = JoinPath({dest, BaseName(blob_item.Name)}); try { - std::string url = container_client.GetBlobClient(blob_item.Name).GetUrl(); + std::string url = + container_client.GetBlobClient(blob_item.Name).GetUrl(); RETURN_IF_ERROR(DownloadFile(url, local_path, config)); } catch (as::StorageException& ex) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - ("Failed to download file at " + blob_item.Name + ":" + ex.what()).c_str()); + ("Failed to download file at " + blob_item.Name + ":" + ex.what()) + .c_str()); } } for (const auto& directory_item : blob_prefixes) { @@ -230,9 +234,11 @@ ASFileSystem::DownloadFolder( return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, ("Failed to create local folder: " + local_path + - ", errno:" + strerror(errno)).c_str()); + ", errno:" + strerror(errno)) + .c_str()); } - RETURN_IF_ERROR(DownloadFolder(container, directory_item, local_path, config)); + RETURN_IF_ERROR( + DownloadFolder(container, directory_item, local_path, config)); } return nullptr; }; @@ -241,7 +247,8 @@ ASFileSystem::DownloadFolder( TRITONSERVER_Error* ASFileSystem::LocalizePath( - const std::string& location, const std::string& temp_dir, DragonflyConfig& config) + const std::string& location, const std::string& temp_dir, + DragonflyConfig& config) { bool exists; RETURN_IF_ERROR(FileExists(location, &exists)); @@ -264,13 +271,12 @@ ASFileSystem::LocalizePath( return DownloadFolder(container, blob, temp_dir, config); } - TRITONSERVER_Error* +TRITONSERVER_Error* ASFileSystem::ListDirectory( const std::string& container, const std::string& dir_path, const std::function& blobs, - const std::vector& blob_prefixes)>& - callback) + const std::vector& blob_prefixes)>& callback) { auto container_client = client_->GetBlobContainerClient(container); auto options = asb::ListBlobsOptions(); @@ -288,7 +294,8 @@ ASFileSystem::ListDirectory( catch (as::StorageException& ex) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - ("Failed to get contents of directory " + dir_path + ":" + ex.what()).c_str()); + ("Failed to get contents of directory " + dir_path + ":" + ex.what()) + .c_str()); } return nullptr; diff --git a/src/filesystem/implementations/common.h b/src/filesystem/implementations/common.h index be5a7d0..8bca96d 100644 --- a/src/filesystem/implementations/common.h +++ b/src/filesystem/implementations/common.h @@ -1,17 +1,17 @@ /* -* Copyright 2023 The Dragonfly Authors -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. + * Copyright 2023 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #pragma once @@ -23,11 +23,10 @@ namespace triton::repoagent::dragonfly { class FileSystem { public: virtual TRITONSERVER_Error* LocalizePath( - const std::string& location, - const std::string& temp_dir, + const std::string& location, const std::string& temp_dir, DragonflyConfig& config) = 0; virtual ~FileSystem() = default; }; -} \ No newline at end of file +} // namespace triton::repoagent::dragonfly \ No newline at end of file diff --git a/src/filesystem/implementations/gcs.h b/src/filesystem/implementations/gcs.h index ba1e1de..d232abf 100644 --- a/src/filesystem/implementations/gcs.h +++ b/src/filesystem/implementations/gcs.h @@ -1,42 +1,42 @@ /* -* Copyright 2023 The Dragonfly Authors -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. + * Copyright 2023 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ #pragma once -#include "google/cloud/storage/client.h" -#include "triton/core/tritonserver.h" -#include "set" - #include "common.h" #include "common_utils.h" -#include "sys/stat.h" -#include "iostream" #include "fstream" +#include "google/cloud/storage/client.h" +#include "iostream" +#include "set" +#include "sys/stat.h" +#include "triton/core/tritonserver.h" -namespace triton::repoagent::dragonfly{ +namespace triton::repoagent::dragonfly { namespace gcs = google::cloud::storage; struct GCSCredential { std::string path_; - explicit GCSCredential(triton::common::TritonJson::Value& cred_json); + explicit GCSCredential(triton::common::TritonJson::Value& cred_json); }; -GCSCredential::GCSCredential(triton::common::TritonJson::Value& cred_json){ +GCSCredential::GCSCredential(triton::common::TritonJson::Value& cred_json) +{ cred_json.AsString(&path_); } @@ -47,11 +47,13 @@ class GCSFileSystem : public FileSystem { TRITONSERVER_Error* CheckClient(); // unify with S3 interface - TRITONSERVER_Error* CheckClient(const std::string& path) { return CheckClient(); } + TRITONSERVER_Error* CheckClient(const std::string& path) + { + return CheckClient(); + } TRITONSERVER_Error* LocalizePath( - const std::string& location, - const std::string& temp_dir, + const std::string& location, const std::string& temp_dir, DragonflyConfig& config) override; private: @@ -61,12 +63,14 @@ class GCSFileSystem : public FileSystem { const std::string& path, std::set* contents); static TRITONSERVER_Error* ParsePath( const std::string& path, std::string* bucket, std::string* object); - std::string GenerateGetSignedUrl(std::string const& bucket_name, std::string const& object_name); + std::string GenerateGetSignedUrl( + std::string const& bucket_name, std::string const& object_name); std::unique_ptr client_; }; -GCSFileSystem::GCSFileSystem(const std::string& path, const GCSCredential& gs_cred) +GCSFileSystem::GCSFileSystem( + const std::string& path, const GCSCredential& gs_cred) { google::cloud::Options options; auto creds = gcs::oauth2::CreateServiceAccountCredentialsFromJsonFilePath( @@ -100,7 +104,6 @@ TRITONSERVER_Error* GCSFileSystem::ParsePath( const std::string& path, std::string* bucket, std::string* object) { - // Get the bucket name and the object path. Return error if input is malformed std::string::size_type bucket_start = path.find("gs://"); if (bucket_start != std::string::npos) { @@ -120,8 +123,7 @@ GCSFileSystem::ParsePath( if (bucket->empty()) { return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INTERNAL, - "No bucket name found in path"); + TRITONSERVER_ERROR_INTERNAL, "No bucket name found in path"); } return nullptr; @@ -143,7 +145,8 @@ GCSFileSystem::GenerateGetSignedUrl( } TRITONSERVER_Error* -GCSFileSystem::FileExists(const std::string& path, bool* exists) { +GCSFileSystem::FileExists(const std::string& path, bool* exists) +{ *exists = false; std::string bucket, object; @@ -212,11 +215,9 @@ GCSFileSystem::GetDirectoryContents( for (auto&& object_metadata : client_->ListObjects(bucket, gcs::Prefix(full_dir))) { if (!object_metadata) { - std::string msg = "Could not list contents of directory at " + - path + " : " + - object_metadata.status().message(); - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INTERNAL, msg.c_str()); + std::string msg = "Could not list contents of directory at " + path + + " : " + object_metadata.status().message(); + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, msg.c_str()); } // In the case of empty directories, the directory itself will appear here @@ -232,7 +233,8 @@ GCSFileSystem::GetDirectoryContents( // Let set take care of subdirectory contents std::string item; - if (item_end != std::string::npos) { // ensure that item_end is valid before substr + if (item_end != + std::string::npos) { // ensure that item_end is valid before substr item = name.substr(item_start, item_end - item_start); } else { item = name.substr(item_start); @@ -244,7 +246,6 @@ GCSFileSystem::GetDirectoryContents( return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, ("Cannot handle item with empty name at " + path).c_str()); - } } return nullptr; @@ -252,7 +253,8 @@ GCSFileSystem::GetDirectoryContents( TRITONSERVER_Error* GCSFileSystem::LocalizePath( - const std::string& location, const std::string& temp_dir, DragonflyConfig& config) + const std::string& location, const std::string& temp_dir, + DragonflyConfig& config) { bool exists; // @@ -282,8 +284,7 @@ GCSFileSystem::LocalizePath( for (const auto& gcs_fpath : tmp_contents) { bool is_subdir; std::string gcs_removed_path = gcs_fpath.substr(location.size()); - std::string local_fpath = - JoinPath({temp_dir, gcs_removed_path}); + std::string local_fpath = JoinPath({temp_dir, gcs_removed_path}); RETURN_IF_ERROR(IsDirectory(gcs_fpath, &is_subdir)); if (is_subdir) { #ifdef _WIN32 @@ -319,4 +320,4 @@ GCSFileSystem::LocalizePath( return nullptr; } -} \ No newline at end of file +} // namespace triton::repoagent::dragonfly \ No newline at end of file diff --git a/src/filesystem/implementations/s3.h b/src/filesystem/implementations/s3.h index a309118..e7e77e7 100644 --- a/src/filesystem/implementations/s3.h +++ b/src/filesystem/implementations/s3.h @@ -1,30 +1,21 @@ /* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #pragma once #include "aws/core/Aws.h" #include "aws/core/auth/AWSCredentialsProvider.h" -#include "re2/re2.h" -#include "triton/core/tritonserver.h" - -#include "common.h" -#include "common_utils.h" -#include "sys/stat.h" -#include "iostream" -#include "fstream" - #include "aws/core/client/ClientConfiguration.h" #include "aws/core/http/curl/CurlHttpClient.h" #include "aws/core/http/standard/StandardHttpRequest.h" @@ -33,8 +24,15 @@ #include "aws/s3/model/HeadObjectRequest.h" #include "aws/s3/model/ListObjectsV2Request.h" #include "aws/s3/model/ListObjectsV2Result.h" +#include "common.h" +#include "common_utils.h" +#include "fstream" +#include "iostream" +#include "re2/re2.h" +#include "sys/stat.h" +#include "triton/core/tritonserver.h" -namespace triton::repoagent::dragonfly{ +namespace triton::repoagent::dragonfly { namespace s3 = Aws::S3; @@ -114,7 +112,8 @@ class S3FileSystem : public FileSystem { S3FileSystem(const std::string& s3_path, const S3Credential& s3_cred); TRITONSERVER_Error* LocalizePath( - const std::string& location, const std::string& temp_dir, DragonflyConfig& config) override; + const std::string& location, const std::string& temp_dir, + DragonflyConfig& config) override; TRITONSERVER_Error* CheckClient(const std::string& s3_path); @@ -125,12 +124,14 @@ class S3FileSystem : public FileSystem { const std::string& path, std::set* contents); TRITONSERVER_Error* ParsePath( const std::string& path, std::string* bucket, std::string* object); - static TRITONSERVER_Error* CleanPath(const std::string& s3_path, std::string* clean_path); + static TRITONSERVER_Error* CleanPath( + const std::string& s3_path, std::string* clean_path); std::unique_ptr client_; // init after Aws::InitAPI is called re2::RE2 s3_regex_; }; -TRITONSERVER_Error* S3FileSystem::ParsePath( +TRITONSERVER_Error* +S3FileSystem::ParsePath( const std::string& path, std::string* bucket, std::string* object) { // Cleanup extra slashes @@ -315,7 +316,8 @@ S3FileSystem::CheckClient(const std::string& s3_path) TRITONSERVER_ERROR_INTERNAL, ("Unable to create S3 filesystem client. Check account credentials. " "Exception: '" + - err.GetExceptionName() + "' Message: '" + err.GetMessage() + "'").c_str()); + err.GetExceptionName() + "' Message: '" + err.GetMessage() + "'") + .c_str()); } return nullptr; } @@ -351,8 +353,8 @@ S3FileSystem::FileExists(const std::string& path, bool* exists) ("Could not get MetaData for object at " + path + " due to exception: " + head_object_outcome.GetError().GetExceptionName() + - ", error message: " + - head_object_outcome.GetError().GetMessage()).c_str()); + ", error message: " + head_object_outcome.GetError().GetMessage()) + .c_str()); } } else { *exists = true; @@ -380,7 +382,8 @@ S3FileSystem::IsDirectory(const std::string& path, bool* is_dir) ("Could not get MetaData for bucket with name " + bucket + " due to exception: " + head_bucket_outcome.GetError().GetExceptionName() + - ", error message: " + head_bucket_outcome.GetError().GetMessage()).c_str()); + ", error message: " + head_bucket_outcome.GetError().GetMessage()) + .c_str()); } // Root case - bucket exists and object path is empty @@ -402,7 +405,8 @@ S3FileSystem::IsDirectory(const std::string& path, bool* is_dir) TRITONSERVER_ERROR_INTERNAL, ("Failed to list objects with prefix " + path + " due to exception: " + list_objects_outcome.GetError().GetExceptionName() + - ", error message: " + list_objects_outcome.GetError().GetMessage()).c_str()); + ", error message: " + list_objects_outcome.GetError().GetMessage()) + .c_str()); } return nullptr; } @@ -434,8 +438,8 @@ S3FileSystem::GetDirectoryContents( ("Could not list contents of directory at " + true_path + " due to exception: " + list_objects_outcome.GetError().GetExceptionName() + - ", error message: " + - list_objects_outcome.GetError().GetMessage()).c_str()); + ", error message: " + list_objects_outcome.GetError().GetMessage()) + .c_str()); } const auto& list_objects_result = list_objects_outcome.GetResult(); for (const auto& s3_object : list_objects_result.GetContents()) { @@ -475,13 +479,15 @@ S3FileSystem::GetDirectoryContents( TRITONSERVER_Error* S3FileSystem::LocalizePath( - const std::string& location, const std::string& temp_dir, DragonflyConfig& config) { + const std::string& location, const std::string& temp_dir, + DragonflyConfig& config) +{ bool exists; RETURN_IF_ERROR(FileExists(location, &exists)); if (!exists) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, - ("directory or file does not exist at " + location).c_str()); + ("directory or file does not exist at " + location).c_str()); } std::string clean_path; @@ -496,11 +502,11 @@ S3FileSystem::LocalizePath( effective_path = location; } - std::set contents; + std::set contents; bool is_dir; RETURN_IF_ERROR(IsDirectory(location, &is_dir)); if (is_dir) { - std::set filenames; + std::set filenames; RETURN_IF_ERROR(GetDirectoryContents(effective_path, &filenames)); for (auto itr = filenames.begin(); itr != filenames.end(); ++itr) { contents.insert(JoinPath({effective_path, *itr})); @@ -513,14 +519,13 @@ S3FileSystem::LocalizePath( } while (!contents.empty()) { - std::set tmp_contents = contents; + std::set tmp_contents = contents; contents.clear(); for (const auto& s3_fpath : tmp_contents) { std::string s3_removed_path = s3_fpath.substr(effective_path.size()); - std::string local_fpath = - s3_removed_path.empty() - ? temp_dir - : JoinPath({temp_dir, s3_removed_path}); + std::string local_fpath = s3_removed_path.empty() + ? temp_dir + : JoinPath({temp_dir, s3_removed_path}); bool is_subdir; RETURN_IF_ERROR(IsDirectory(s3_fpath, &is_subdir)); if (is_subdir) { @@ -535,12 +540,13 @@ S3FileSystem::LocalizePath( return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INTERNAL, ("Failed to create local folder: " + local_fpath + - ", errno:" + strerror(errno)).c_str()); + ", errno:" + strerror(errno)) + .c_str()); } - std::set subdir_contents; + std::set subdir_contents; RETURN_IF_ERROR(GetDirectoryContents(s3_fpath, &subdir_contents)); - for (const auto & subdir_content : subdir_contents) { + for (const auto& subdir_content : subdir_contents) { contents.insert(JoinPath({s3_fpath, subdir_content})); } } else { @@ -548,7 +554,7 @@ S3FileSystem::LocalizePath( RETURN_IF_ERROR(ParsePath(s3_fpath, &file_bucket, &file_object)); std::string url = client_->GeneratePresignedUrl( - file_bucket,file_object, Aws::Http::HttpMethod::HTTP_GET); + file_bucket, file_object, Aws::Http::HttpMethod::HTTP_GET); RETURN_IF_ERROR(DownloadFile(url, local_fpath, config)); } } @@ -557,4 +563,4 @@ S3FileSystem::LocalizePath( return nullptr; } -} // namespace triton::core +} // namespace triton::repoagent::dragonfly diff --git a/src/status.h b/src/status.h index b24cad0..7a7b29b 100644 --- a/src/status.h +++ b/src/status.h @@ -8,11 +8,11 @@ struct ErrorException { TRITONSERVER_Error* err_; }; -#define RETURN_IF_ERROR(X) \ - do { \ - TRITONSERVER_Error* rie_err__ = (X); \ - if (rie_err__ != nullptr) { \ - return rie_err__; \ - } \ - } while (false) -#endif // STATUS_H \ No newline at end of file +#define RETURN_IF_ERROR(X) \ + do { \ + TRITONSERVER_Error* rie_err__ = (X); \ + if (rie_err__ != nullptr) { \ + return rie_err__; \ + } \ + } while (false) +#endif // STATUS_H From c0a3b0b1d9b987ecc8c80a1ab929ca66da7fead2 Mon Sep 17 00:00:00 2001 From: chenyufeifei Date: Sat, 4 Nov 2023 22:43:47 +0800 Subject: [PATCH 4/5] Add GitHub workflow to build shared library Signed-off-by: chenyufeifei --- .github/workflows/build.yml | 92 ++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 53 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5121e00..c499adc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,14 +2,13 @@ name: Build on: push: - branches: - - main - tags: - - v* + branches: [main, release-*] + pull_request: + branches: [main, release-*] jobs: - push_image_to_registry: - name: Push Image + build: + name: Build libtritonrepoagent_dragonfly.so runs-on: ubuntu-latest steps: - name: Check out code @@ -17,56 +16,43 @@ jobs: with: submodules: recursive - - name: Get Version - id: get_version + - name: Install dependencies run: | - VERSION=${GITHUB_REF#refs/tags/} - if [[ ${GITHUB_REF} == "refs/heads/main" ]]; then - VERSION=latest - fi - echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT - - - name: Get Git Revision - id: vars - shell: bash + sudo apt-get update && sudo apt-get install -y \ + build-essential \ + cmake \ + git \ + curl \ + zip \ + unzip \ + tar \ + pkg-config \ + python3 + + - name: Install vcpkg run: | - echo "git_revision=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + git clone https://github.com/Microsoft/vcpkg.git $HOME/vcpkg + $HOME/vcpkg/bootstrap-vcpkg.sh - - name: PrepareReg Names + - name: Install project dependencies with vcpkg run: | - echo IMAGE_REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV - - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3 + $HOME/vcpkg/vcpkg install re2 + $HOME/vcpkg/vcpkg install aws-sdk-cpp + $HOME/vcpkg/vcpkg install google-cloud-cpp[storage] + $HOME/vcpkg/vcpkg install azure-storage-blobs-cpp + $HOME/vcpkg/vcpkg install rapidjson - - name: Login Docker Hub - uses: docker/login-action@v3 - with: - registry: docker.io - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + - name: Create build directory + run: mkdir -p $HOME/build - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push to Registry - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64 - file: Dockerfile - labels: |- - org.opencontainers.image.source=https://github.com/${{ github.repository }} - org.opencontainers.image.revision=${{ github.sha }} - build-args: | - GITVERSION=git-${{ steps.vars.outputs.git_revision }} - VERSION=${{ steps.get_version.outputs.VERSION }} - tags: | - dragonflyoss/tritonserver-dragonfly:${{ steps.get_version.outputs.VERSION }} - ghcr.io/${{ env.IMAGE_REPOSITORY }}/tritonserver-dragonfly:${{ steps.get_version.outputs.VERSION }} - push: - true \ No newline at end of file + - name: Build the repository agent + run: | + mkdir -p $HOME/build && cd $HOME/build + cmake -DCMAKE_TOOLCHAIN_FILE=$HOME/vcpkg/scripts/buildsystems/vcpkg.cmake \ + -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install \ + -DTRITON_ENABLE_GCS=true \ + -DTRITON_ENABLE_AZURE_STORAGE=true \ + -DTRITON_ENABLE_S3=true \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ + $GITHUB_WORKSPACE + make install From 7be08ed05a64ffd3a7b698bd8dcdd855f0df5827 Mon Sep 17 00:00:00 2001 From: chenyufeifei Date: Sun, 5 Nov 2023 13:26:00 +0800 Subject: [PATCH 5/5] add a blank line Signed-off-by: chenyufeifei --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a25bf0a..89deec1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,4 +27,4 @@ jobs: echo "Code style issues found" git diff exit 1 - fi \ No newline at end of file + fi