kettle: migrate off of gs://kubernetes-jenkins

Switch the gs bucket referenced from `kubernetes-jenkins` -> `kubernetes-ci-logs` updates #33381 Signed-off-by: jbpratt <[email protected]>
kubernetes · Sep 17, 2024 · a5f882e · a5f882e
1 parent 6a352c5
commit a5f882e
Show file tree

Hide file tree

Showing 7 changed files with 62 additions and 62 deletions.
diff --git a/kettle/README.md b/kettle/README.md
@@ -40,8 +40,8 @@ kubectl logs -l app=kettle
 PULLED 174
 ACK irrelevant 172
 EXTEND-ACK  2
-gs://kubernetes-jenkins/pr-logs/pull/kubeflow_kubeflow/1136/kubeflow-presubmit/2385 True True 2018-07-06 07:51:49 PDT FAILED
-gs://kubernetes-jenkins/logs/ci-cri-containerd-e2e-ubuntu-gce/5742 True True 2018-07-06 07:44:17 PDT FAILURE
+gs://kubernetes-ci-logs/pr-logs/pull/kubeflow_kubeflow/1136/kubeflow-presubmit/2385 True True 2018-07-06 07:51:49 PDT FAILED
+gs://kubernetes-ci-logs/logs/ci-cri-containerd-e2e-ubuntu-gce/5742 True True 2018-07-06 07:44:17 PDT FAILURE
 ACK "finished.json" 2
 Downloading JUnit artifacts.
 ```
@@ -102,9 +102,9 @@ A [postsubmit job](https://github.com/kubernetes/test-infra/blob/master/config/j
 
 # PubSub
 
-Kettle `stream.py` leverages Google Cloud [PubSub] to alert on GCS changes within the `kubernetes-jenkins` bucket. These events are tied to the `gcs-changes` Topic in the `kubernetes-jenkins` project where Prow job artifacts are collated. Each time an artifact is finalized, a PubSub event is triggered and Kettle collects job information when it sees a resource uploaded called `finished.json` (indicating the build completed).
+Kettle `stream.py` leverages Google Cloud [PubSub] to alert on GCS changes within the `kubernetes-ci-logs` bucket. These events are tied to the `gcs-changes` Topic in the `kubernetes-ci-logs` project where Prow job artifacts are collated. Each time an artifact is finalized, a PubSub event is triggered and Kettle collects job information when it sees a resource uploaded called `finished.json` (indicating the build completed).
 
-[Topic Creation] can be performed by running `gcloud config set project kubernetes-jenkins` and `gsutil notification create -t gcs-changes -f json gs://kubernetes-jenkins`
+[Topic Creation] can be performed by running `gcloud config set project kubernetes-ci-logs` and `gsutil notification create -t gcs-changes -f json gs://kubernetes-ci-logs`
 
 [Subscriptions] are in Kuberenetes Jenkins Build - PubSub.
 - kettle
@@ -114,14 +114,14 @@ They are split so that the staging instance does not consume events aimed at pro
 
 These can be created via:
 ```
-gcloud pubsub subscriptions create <subscription name> --topic=gcs-changes --topic-project="kubernetes-jenkins" --message-filter='attributes.eventType = "OBJECT_FINALIZE"'
+gcloud pubsub subscriptions create <subscription name> --topic=gcs-changes --topic-project="kubernetes-ci-logs" --message-filter='attributes.eventType = "OBJECT_FINALIZE"'
 ```
 
 ### Auth
 For kettle to have permission, kettle's user needs access. When updating or changing a [Subscription] make sure to add `[email protected]` as a `PubSub Editor`.
 ```
 gcloud pubsub subscriptions add-iam-policy-binding \
-  projects/kubernetes-jenkins/subscriptions/kettle-staging \
+  projects/kubernetes-ci-logs/subscriptions/kettle-staging \
   --member=serviceAccount:kettle@kubernetes-public.iam.gserviceaccount.com \
   --role=roles/pubsub.editor
 ```
@@ -134,5 +134,5 @@ gcloud pubsub subscriptions add-iam-policy-binding \
 [Big Query All]: https://console.cloud.google.com/bigquery?project=kubernetes-public&page=table&t=all&d=build&p=kubernetes-public
 [Big Query Staging]: https://console.cloud.google.com/bigquery?project=kubernetes-public&page=table&t=staging&d=build&p=kubernetes-public
 [PubSub]: https://cloud.google.com/pubsub/docs
-[Subscriptions]: https://console.cloud.google.com/cloudpubsub/subscription/list?project=kubernetes-jenkins
+[Subscriptions]: https://console.cloud.google.com/cloudpubsub/subscription/list?project=kubernetes-ci-logs
 [Topic Creation]: https://cloud.google.com/storage/docs/reporting-changes#enabling
diff --git a/kettle/buckets.yaml b/kettle/buckets.yaml
@@ -8,7 +8,7 @@
 #     collection phase, and defaults to true.
 #   exclude_jobs: A list of job names that will not be uploaded to BQ
 
-gs://kubernetes-jenkins/logs/:
+gs://kubernetes-ci-logs/logs/:
   contact: "bentheelder"
   prefix: ""
   sequential: false

diff --git a/kettle/make_db.py b/kettle/make_db.py
@@ -329,7 +329,7 @@ def main(db, jobs_dirs, threads, get_junit, build_limit, skip_gcs, client_class=
     """Collect test info in matching jobs."""
     setup_logging()
     if not skip_gcs:
-        get_all_builds(db, 'gs://kubernetes-jenkins/pr-logs', {'pr': True},
+        get_all_builds(db, 'gs://kubernetes-ci-logs/pr-logs', {'pr': True},
                        threads, client_class, build_limit)
         for bucket, metadata in jobs_dirs.items():
             if not bucket.endswith('/'):

diff --git a/kettle/make_db_test.py b/kettle/make_db_test.py
@@ -25,7 +25,7 @@
 static_epoch = 1641585162
 
 TEST_BUCKETS_DATA = {
-    'gs://kubernetes-jenkins/logs/': {'prefix': ''},
+    'gs://kubernetes-ci-logs/logs/': {'prefix': ''},
     'gs://bucket1/': {'prefix': 'bucket1_prefix'},
     'gs://bucket2/': {'prefix': 'bucket2_prefix'}
 }
@@ -34,15 +34,15 @@
 class MockedClient(make_db.GCSClient):
     """A GCSClient with stubs for external interactions."""
     NOW = static_epoch
-    LOG_DIR = 'gs://kubernetes-jenkins/logs/'
+    LOG_DIR = 'gs://kubernetes-ci-logs/logs/'
     JOB_DIR = LOG_DIR + 'fake/123/'
     ART_DIR = JOB_DIR + 'artifacts/'
     lists = {
         LOG_DIR: [LOG_DIR + 'fake/'],
         LOG_DIR + 'fake/': [JOB_DIR, LOG_DIR + 'fake/122/'],
         LOG_DIR + 'bad-latest/': [LOG_DIR + 'bad-latest/6/'],
         LOG_DIR + 'latest/': [LOG_DIR + 'latest/4/', LOG_DIR + 'latest/3/'],
-        'gs://kubernetes-jenkins/pr-logs/directory/': [],
+        'gs://kubernetes-ci-logs/pr-logs/directory/': [],
         ART_DIR: [ART_DIR + 'junit_01.xml'],
         ART_DIR.replace('123', '122'): [],
     }
@@ -76,7 +76,7 @@ class GCSClientTest(unittest.TestCase):
 
     # pylint: disable=protected-access
 
-    JOBS_DIR = 'gs://kubernetes-jenkins/logs/'
+    JOBS_DIR = 'gs://kubernetes-ci-logs/logs/'
 
     def setUp(self):
         self.client = MockedClient(self.JOBS_DIR)
@@ -85,7 +85,7 @@ def test_get_junits(self):
         junits = self.client.get_junits_from_build(self.JOBS_DIR + 'fake/123')
         self.assertEqual(
             sorted(junits),
-            ['gs://kubernetes-jenkins/logs/fake/123/artifacts/junit_01.xml'])
+            ['gs://kubernetes-ci-logs/logs/fake/123/artifacts/junit_01.xml'])
 
     def test_get_builds_normal_list(self):
         # normal case: lists a directory
@@ -184,14 +184,14 @@ def test_incremental_new(self):
 
         class MockedClientNewer(MockedClient):
             NOW = static_epoch
-            LOG_DIR = 'gs://kubernetes-jenkins/logs/'
+            LOG_DIR = 'gs://kubernetes-ci-logs/logs/'
             JOB_DIR = LOG_DIR + 'fake/124/'
             ART_DIR = JOB_DIR + 'artifacts/'
             lists = {
                 LOG_DIR: [LOG_DIR + 'fake/'],
                 LOG_DIR + 'fake/': [JOB_DIR, LOG_DIR + 'fake/123/'],
                 ART_DIR: [ART_DIR + 'junit_01.xml'],
-                'gs://kubernetes-jenkins/pr-logs/directory/': [],
+                'gs://kubernetes-ci-logs/pr-logs/directory/': [],
             }
             gets = {
                 JOB_DIR + 'finished.json': {'timestamp': NOW},

diff --git a/kettle/make_json.py b/kettle/make_json.py
@@ -115,9 +115,9 @@ def populate_path_to_job_and_number(self):
             if self.path.startswith(bucket):
                 prefix = meta['prefix']
                 break
-        #if job path not in buckets.yaml or gs://kubernetes-jenkins/pr-logs it is unmatched
+        #if job path not in buckets.yaml or gs://kubernetes-ci-logs/pr-logs it is unmatched
         else:
-            if self.path.startswith('gs://kubernetes-jenkins/pr-logs'):
+            if self.path.startswith('gs://kubernetes-ci-logs/pr-logs'):
                 prefix = 'pr:'
             else:
                 raise ValueError(f'unknown build path for {self.path} in known bucket paths')

diff --git a/kettle/make_json_test.py b/kettle/make_json_test.py
@@ -29,7 +29,7 @@ class ValidateBuckets(unittest.TestCase):
     def test_buckets(self):
         prefixes = set()
         for name, options in sorted(make_json.BUCKETS.items()):
-            if name == 'gs://kubernetes-jenkins/logs/':
+            if name == 'gs://kubernetes-ci-logs/logs/':
                 continue  # only bucket without a prefix
             prefix = options.get('prefix', '')
             self.assertNotEqual(prefix, '', 'bucket %s must have a prefix' % name)
@@ -46,8 +46,8 @@ class BuildObjectTests(unittest.TestCase):
         ),
         (
             "Base_Build",
-            make_json.Build("gs://kubernetes-jenkins/pr-logs/path", []),
-            {"path":"gs://kubernetes-jenkins/pr-logs/path",
+            make_json.Build("gs://kubernetes-ci-logs/pr-logs/path", []),
+            {"path":"gs://kubernetes-ci-logs/pr-logs/path",
              "test": [],
              "tests_run": 0,
              "tests_failed": 0,
@@ -57,10 +57,10 @@ class BuildObjectTests(unittest.TestCase):
         (
             "Tests_populate",
             make_json.Build(
-                "gs://kubernetes-jenkins/pr-logs/path",
+                "gs://kubernetes-ci-logs/pr-logs/path",
                 [{'name': 'Test1', 'failed': True}],
             ),
-            {"path":"gs://kubernetes-jenkins/pr-logs/path",
+            {"path":"gs://kubernetes-ci-logs/pr-logs/path",
              "test": [{'name': 'Test1', 'failed': True}],
              "tests_run": 1,
              "tests_failed": 1,
@@ -143,9 +143,9 @@ def test_as_dict(self, _, build, expected):
         ),
     ])
     def test_populate_start(self, _, started, updates):
-        build = make_json.Build("gs://kubernetes-jenkins/pr-logs/path", [])
+        build = make_json.Build("gs://kubernetes-ci-logs/pr-logs/path", [])
         attrs = {
-            "path":"gs://kubernetes-jenkins/pr-logs/path",
+            "path":"gs://kubernetes-ci-logs/pr-logs/path",
             "test": [],
             "tests_run": 0,
             "tests_failed": 0,
@@ -223,8 +223,8 @@ def test_populate_start(self, _, started, updates):
         ),
     ])
     def test_populate_finish(self, _, finished, updates):
-        build = make_json.Build("gs://kubernetes-jenkins/pr-logs/path", [])
-        attrs = {"path":"gs://kubernetes-jenkins/pr-logs/path",
+        build = make_json.Build("gs://kubernetes-ci-logs/pr-logs/path", [])
+        attrs = {"path":"gs://kubernetes-ci-logs/pr-logs/path",
                  "test": [],
                  "tests_run": 0,
                  "tests_failed": 0,
@@ -239,39 +239,39 @@ class GenerateBuilds(unittest.TestCase):
     @parameterized.expand([
         (
             "Basic_pass",
-            "gs://kubernetes-jenkins/pr-logs/path",
+            "gs://kubernetes-ci-logs/pr-logs/path",
             [{'name': "Test1", 'failed': False}],
             None,
             None,
             None,
             None,
             {
                 'job': 'pr:pr-logs',
-                'path': 'gs://kubernetes-jenkins/pr-logs/path',
+                'path': 'gs://kubernetes-ci-logs/pr-logs/path',
                 'test': [{'name': 'Test1', 'failed': False}],
                 'tests_run': 1,
                 'tests_failed':0,
             },
         ),
         (
             "Basic_fail",
-            "gs://kubernetes-jenkins/pr-logs/path",
+            "gs://kubernetes-ci-logs/pr-logs/path",
             [{'name': "Test1", 'failed': True}],
             None,
             None,
             None,
             None,
             {
                 'job': 'pr:pr-logs',
-                'path': 'gs://kubernetes-jenkins/pr-logs/path',
+                'path': 'gs://kubernetes-ci-logs/pr-logs/path',
                 'test': [{'name': 'Test1', 'failed': True}],
                 'tests_run': 1,
                 'tests_failed':1,
             },
         ),
         (
             "Ci_decorated",
-            "gs://kubernetes-jenkins/pr-logs/path",
+            "gs://kubernetes-ci-logs/pr-logs/path",
             [{'name': "Test1", 'failed': True}],
             {
                 "timestamp":1595284709,
@@ -289,7 +289,7 @@ class GenerateBuilds(unittest.TestCase):
             None,
             {
                 'job': 'pr:pr-logs',
-                'path': 'gs://kubernetes-jenkins/pr-logs/path',
+                'path': 'gs://kubernetes-ci-logs/pr-logs/path',
                 'test': [{'name': 'Test1', 'failed': True}],
                 'passed': True,
                 'result': 'SUCCESS',
@@ -304,7 +304,7 @@ class GenerateBuilds(unittest.TestCase):
         ),
         (
             "Pr_decorated",
-            "gs://kubernetes-jenkins/pr-logs/path",
+            "gs://kubernetes-ci-logs/pr-logs/path",
             [{'name': "Test1", 'failed': True}],
             {
                 "timestamp":1595277241,
@@ -323,7 +323,7 @@ class GenerateBuilds(unittest.TestCase):
             None,
             {
                 'job': 'pr:pr-logs',
-                'path': 'gs://kubernetes-jenkins/pr-logs/path',
+                'path': 'gs://kubernetes-ci-logs/pr-logs/path',
                 'test': [{'name': 'Test1', 'failed': True}],
                 'passed': True,
                 'result': 'SUCCESS',
@@ -338,7 +338,7 @@ class GenerateBuilds(unittest.TestCase):
         ),
         (
             "Pr_bootstrap",
-            "gs://kubernetes-jenkins/pr-logs/path",
+            "gs://kubernetes-ci-logs/pr-logs/path",
             [{'name': "Test1", 'failed': True}],
             {
                 "node": "0790211c-cacb-11ea-a4b9-4a19d9b965b2",
@@ -371,7 +371,7 @@ class GenerateBuilds(unittest.TestCase):
             {
                 'job': 'pr:pr-logs',
                 'executor': '0790211c-cacb-11ea-a4b9-4a19d9b965b2',
-                'path': 'gs://kubernetes-jenkins/pr-logs/path',
+                'path': 'gs://kubernetes-ci-logs/pr-logs/path',
                 'test': [{'name': 'Test1', 'failed': True}],
                 'passed': True,
                 'result': 'SUCCESS',
@@ -393,7 +393,7 @@ class GenerateBuilds(unittest.TestCase):
         ),
         (
             "Ci_bootstrap",
-            "gs://kubernetes-jenkins/pr-logs/path",
+            "gs://kubernetes-ci-logs/pr-logs/path",
             [{'name': "Test1", 'failed': True}],
             {
                 "timestamp":1595263104,
@@ -427,7 +427,7 @@ class GenerateBuilds(unittest.TestCase):
             {
                 'job': 'pr:pr-logs',
                 'executor': '592473ae-caa7-11ea-b130-525df2b76a8d',
-                'path': 'gs://kubernetes-jenkins/pr-logs/path',
+                'path': 'gs://kubernetes-ci-logs/pr-logs/path',
                 'test': [{'name': 'Test1', 'failed': True}],
                 'passed': True,
                 'result': 'SUCCESS',
@@ -452,7 +452,7 @@ class GenerateBuilds(unittest.TestCase):
         ),
         (
             "Started_no_meta_repo",
-            "gs://kubernetes-jenkins/pr-logs/path",
+            "gs://kubernetes-ci-logs/pr-logs/path",
             [{'name': "Test1", 'failed': False}],
             {
                 "timestamp":1595263104,
@@ -469,7 +469,7 @@ class GenerateBuilds(unittest.TestCase):
             {
                 'job': 'pr:pr-logs',
                 'executor': '592473ae-caa7-11ea-b130-525df2b76a8d',
-                'path': 'gs://kubernetes-jenkins/pr-logs/path',
+                'path': 'gs://kubernetes-ci-logs/pr-logs/path',
                 'test': [{'name': 'Test1', 'failed': False}],
                 'tests_run': 1,
                 'tests_failed':0,
@@ -495,9 +495,9 @@ def expect(path, job, number):
             build = make_json.Build(path, [])
             self.assertEqual((build.job, build.number), (job, number))
 
-        expect('gs://kubernetes-jenkins/logs/some-build/123', 'some-build', 123)
-        expect('gs://kubernetes-jenkins/logs/some-build/123asdf', 'some-build', None)
-        expect('gs://kubernetes-jenkins/pr-logs/123/e2e-node/456', 'pr:e2e-node', 456)
+        expect('gs://kubernetes-ci-logs/logs/some-build/123', 'some-build', 123)
+        expect('gs://kubernetes-ci-logs/logs/some-build/123asdf', 'some-build', None)
+        expect('gs://kubernetes-ci-logs/pr-logs/123/e2e-node/456', 'pr:e2e-node', 456)
 
         with self.assertRaises(ValueError):
             expect('gs://unknown-bucket/foo/123', None, None)
@@ -517,7 +517,7 @@ def expect(path, start, finish, results, **kwargs):
             row = make_json.row_for_build(path, start, finish, results)
             self.assertEqual(row, expected)
 
-        path = 'gs://kubernetes-jenkins/logs/J/123'
+        path = 'gs://kubernetes-ci-logs/logs/J/123'
         expect(path, None, None, [], job='J', number=123)
         expect(path, None, None, [], job='J', number=123)
         expect(path,
@@ -559,7 +559,7 @@ def test_main(self):
         junits = ['<testsuite><testcase name="t1" time="3.0"></testcase></testsuite>']
 
         def add_build(path, start, finish, result, junits):
-            path = 'gs://kubernetes-jenkins/logs/%s' % path
+            path = 'gs://kubernetes-ci-logs/logs/%s' % path
             self.db.insert_build(
                 path, {'timestamp': start}, {'timestamp': finish, 'result': result})
             # fake build rowid doesn't matter here
@@ -607,8 +607,8 @@ def expect(args, needles, negneedles, expected_ret=None):
         expect([], [], ['123', '456', '457'])                     # reset only works for given day
 
         # verify that direct paths work
-        expect(['gs://kubernetes-jenkins/logs/some-job/123'], ['123'], [])
-        expect(['gs://kubernetes-jenkins/logs/some-job/123'], ['123'], [])
+        expect(['gs://kubernetes-ci-logs/logs/some-job/123'], ['123'], [])
+        expect(['gs://kubernetes-ci-logs/logs/some-job/123'], ['123'], [])
 
         # verify that assert_oldest works
         expect(['--days=30'], ['123', '456'], [])