Merge pull request #1281 from egibs/fix-pypi-release-resolution

Add flag to preserve original PyPI URIs

docs/md/melange_convert_python.md

@@ -33,6 +33,7 @@ convert python botocore
       --base-uri-format string   URI to use for querying gems for provided package name (default "https://pypi.org")
   -h, --help                     help for python
       --package-version string   version of the python package to convert
+      --preserve-base-uri        preserve the base URI for PyPI packages instead of using the friendly URL
       --python-version string    version of the python to build the package (default "3")
 ```
 

pkg/cli/python.go

@@ -36,6 +36,7 @@ type pythonOptions struct {
 	packageVersion         string
 	ghClient               *github.Client
 	mf                     *relmon.MonitorFinder
+	preserveBaseURI        bool
 }
 
 // PythonBuild is the top-level `convert python` cobra command
@@ -79,6 +80,7 @@ convert python botocore`,
 	cmd.Flags().StringVar(&o.baseURIFormat, "base-uri-format", "https://pypi.org",
 		"URI to use for querying gems for provided package name")
 	cmd.Flags().StringVar(&o.pythonVersion, "python-version", "3", "version of the python to build the package")
+	cmd.Flags().BoolVar(&o.preserveBaseURI, "preserve-base-uri", false, "preserve the base URI for PyPI packages instead of using the friendly URL")
 	return cmd
 }
 
@@ -97,6 +99,7 @@ func (o pythonOptions) pythonBuild(ctx context.Context, packageName string) erro
 	pythonContext.PackageVersion = o.packageVersion
 	pythonContext.PythonVersion = o.pythonVersion
 	pythonContext.PackageName = packageName
+	pythonContext.PreserveBaseURI = o.preserveBaseURI
 
 	// These two are conditionally set above, and if nil, they are unused.
 	pythonContext.GithubClient = o.ghClient

pkg/convert/python/e2e_test.go

@@ -94,3 +94,74 @@ func TestGenerateManifest(t *testing.T) {
 		assert.Equal(t, got.Pipeline[2].Uses, "strip")
 	}
 }
+
+func TestGenerateManifestPreserveURI(t *testing.T) {
+	ctx := slogtest.TestContextWithLogger(t)
+
+	for i := range versions {
+		pythonctxs, err := SetupContextPreserveURI(versions[i])
+		assert.NoError(t, err)
+
+		// typing-extensions ctx
+		pythonctx := pythonctxs[0]
+		// Add additional repositories and keyrings
+		pythonctx.AdditionalRepositories = []string{"https://packages.wolfi.dev/os"}
+		pythonctx.AdditionalKeyrings = []string{"https://packages.wolfi.dev/os/wolfi-signing.rsa.pub"}
+		got, err := pythonctx.generateManifest(ctx, pythonctx.Package, pythonctx.PackageVersion, nil, nil)
+		assert.NoError(t, err)
+
+		// Check Package
+		assert.Equal(t, got.Package.Name, "py"+versions[i]+"-typing-extensions")
+		assert.Equal(t, got.Package.Version, "4.12.2")
+		assert.EqualValues(t, got.Package.Epoch, 0)
+		assert.Equal(t, got.Package.Description,
+			"Backported and Experimental Type Hints for Python 3.8+",
+		)
+		assert.Equal(t, got.Package.Dependencies.Runtime,
+			[]string{
+				"python-" + versions[i],
+			},
+		)
+
+		// Check Package.Copyright
+		assert.Equal(t, len(got.Package.Copyright), 1)
+		assert.Equal(t, got.Package.Copyright[0].License, "")
+
+		// Check Environment
+		assert.Equal(t, got.Environment.Contents.Packages, []string{
+			"build-base",
+			"busybox",
+			"ca-certificates-bundle",
+			"wolfi-base",
+		})
+
+		// Check Pipeline
+		assert.Equal(t, len(got.Pipeline), 3)
+
+		// Check Pipeline - fetch
+		assert.Equal(t, got.Pipeline[0].Uses, "fetch")
+
+		releases, ok := pythonctx.Package.Releases[pythonctx.PackageVersion]
+
+		// If the key exists
+		assert.True(t, ok)
+
+		var release Release
+		for _, r := range releases {
+			if r.PackageType == "sdist" {
+				release = r
+			}
+		}
+		assert.NotEmpty(t, release)
+		assert.Equal(t, "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-"+pythonctx.PackageVersion+".tar.gz", release.URL)
+
+		assert.Equal(t, got.Pipeline[0].With, map[string]string{
+			"expected-sha256": "1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8",
+			"uri":             "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-${{package.version}}.tar.gz",
+		})
+
+		// Check Pipeline - runs
+		assert.Equal(t, got.Pipeline[1].Uses, "python/build-wheel")
+		assert.Equal(t, got.Pipeline[2].Uses, "strip")
+	}
+}

pkg/convert/python/python.go

@@ -34,7 +34,6 @@ import (
 
 // PythonContext is the execution context for the python subcommand.
 type PythonContext struct {
-
 	// PackageName is the name of the python package to build and install
 	PackageName string
 
@@ -80,6 +79,10 @@ type PythonContext struct {
 	// If non-nil, this is the Release Monitoring client to use for fetching
 	// metadata to get the monitoring data for the package.
 	MonitoringClient *relmon.MonitorFinder
+
+	// If true, avoid converting the PyPI URI to a friendly, human-readable URL
+	// May help with conversion failures (404s)
+	PreserveBaseURI bool
 }
 
 // New initialises a new PythonContext.
@@ -400,7 +403,7 @@ func (c *PythonContext) generatePipeline(ctx context.Context, pack Package, vers
 
 		releaseURL := release.URL
 		uri := strings.ReplaceAll(releaseURL, version, "${{package.version}}")
-		if strings.Contains(release.URL, "https://files.pythonhosted.org") {
+		if strings.Contains(release.URL, "https://files.pythonhosted.org") && !c.PreserveBaseURI {
 			packageName := strings.TrimPrefix(pack.Info.Name, fmt.Sprintf("py%s", release.PythonVersion))
 			releaseURL = fmt.Sprintf("https://files.pythonhosted.org/packages/source/%c/%s/%s-%s.tar.gz", packageName[0], packageName, packageName, version)
 
@@ -436,7 +439,8 @@ func (c *PythonContext) generatePipeline(ctx context.Context, pack Package, vers
 				"repository":      ghVersion.Repo,
 				"tag":             ghVersion.TagPrefix + "${{package.version}}",
 				"expected-commit": ghVersion.SHA,
-			}})
+			},
+		})
 	}
 
 	pythonBuild := config.Pipeline{

pkg/convert/python/python_test.go

@@ -36,6 +36,7 @@ const (
 	testDataDir    = "testdata/"
 	botocoreMeta   = testDataDir + "/meta/pypi/botocore/"
 	jsonschemaMeta = testDataDir + "/meta/pypi/jsonschema/"
+	typingextMeta  = testDataDir + "/meta/pypi/typing-extensions/"
 	pypiMetaDir    = testDataDir + "/meta"
 )
 
@@ -224,11 +225,47 @@ func SetupContext(version string) ([]*PythonContext, error) {
 	return pythonctxs, nil
 }
 
+func SetupContextPreserveURI(version string) ([]*PythonContext, error) {
+	typingextctx, err := New("typing-extensions")
+	if err != nil {
+		return nil, err
+	}
+
+	typingextctx.PackageIndex = NewPackageIndex("https://pypi.org/")
+	typingextctx.PackageIndex.Client.Ratelimiter = nil // don't rate limit our unit tests
+	typingextctx.PackageName = "typing-extensions"
+	typingextctx.PackageVersion = "4.12.2"
+	typingextctx.PythonVersion = version
+	typingextctx.PreserveBaseURI = true
+
+	// Read the gem meta into
+	data, err := os.ReadFile(filepath.Join(typingextMeta, "json"))
+	if err != nil {
+		return nil, err
+	}
+
+	var typingextPackageMeta Package
+	err = json.Unmarshal(data, &typingextPackageMeta)
+	if err != nil {
+		return nil, err
+	}
+
+	typingextctx.Package = typingextPackageMeta
+	typingextctx.Package.Dependencies = []string{}
+
+	pythonctxs := []*PythonContext{
+		&typingextctx,
+	}
+	return pythonctxs, nil
+}
+
 func GetJsonsPackagesForPackage(packageName string) ([]string, error) {
 	if packageName == "botocore" {
 		return []string{"botocore", "jmespath", "python-dateutil", "urllib3", "six"}, nil
 	} else if packageName == "jsonschema" {
 		return []string{"jsonschema", "attrs", "importlib-metadata", "importlib-resources", "pkgutil_resolve_name", "pyrsistent", "typing-extensions", "zipp"}, nil
+	} else if packageName == "typing-extensions" {
+		return []string{"typing-extensions"}, nil
 	}
 	return nil, fmt.Errorf("Unknown package %s", packageName)
 }