diff --git a/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3/foo.bar.1.2.3.pom b/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3/foo.bar.1.2.3.pom new file mode 100644 index 0000000..3745968 --- /dev/null +++ b/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3/foo.bar.1.2.3.pom @@ -0,0 +1,7 @@ + + + 4.0.0 + foo + bar + 1.2.3 + diff --git a/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3/foo.bar.1.2.3.rhlw-00001.jar b/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3/foo.bar.1.2.3.rhlw-00001.jar new file mode 100644 index 0000000..e979a54 Binary files /dev/null and b/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3/foo.bar.1.2.3.rhlw-00001.jar differ diff --git a/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3/foo.bar.1.2.3.rhlw-00002.jar b/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3/foo.bar.1.2.3.rhlw-00002.jar new file mode 100644 index 0000000..e979a54 Binary files /dev/null and b/compose_files/pulp/assets/maven-fixture/foo/bar/1.2.3/foo.bar.1.2.3.rhlw-00002.jar differ diff --git a/compose_files/pulp/assets/maven-fixture/foo/bar/2.0.1/foo.bar.2.0.1.pom b/compose_files/pulp/assets/maven-fixture/foo/bar/2.0.1/foo.bar.2.0.1.pom new file mode 100644 index 0000000..29f7623 --- /dev/null +++ b/compose_files/pulp/assets/maven-fixture/foo/bar/2.0.1/foo.bar.2.0.1.pom @@ -0,0 +1,7 @@ + + + 4.0.0 + foo + bar + 2.0.1 + diff --git a/compose_files/pulp/assets/maven-fixture/foo/bar/2.0.1/foo.bar.2.0.1.rhlw-00001.jar b/compose_files/pulp/assets/maven-fixture/foo/bar/2.0.1/foo.bar.2.0.1.rhlw-00001.jar new file mode 100644 index 0000000..e979a54 Binary files /dev/null and b/compose_files/pulp/assets/maven-fixture/foo/bar/2.0.1/foo.bar.2.0.1.rhlw-00001.jar differ diff --git a/compose_files/pulp/assets/maven-fixture/junit/junit/4.13.2/junit-4.13.2.pom b/compose_files/pulp/assets/maven-fixture/junit/junit/4.13.2/junit-4.13.2.pom deleted file mode 100644 index 38be960..0000000 --- a/compose_files/pulp/assets/maven-fixture/junit/junit/4.13.2/junit-4.13.2.pom +++ /dev/null @@ -1,633 +0,0 @@ - - - 4.0.0 - - junit - junit - 4.13.2 - - JUnit - JUnit is a unit testing framework for Java, created by Erich Gamma and Kent Beck. - http://junit.org - 2002 - - JUnit - http://www.junit.org - - - - Eclipse Public License 1.0 - http://www.eclipse.org/legal/epl-v10.html - repo - - - - - - dsaff - David Saff - david@saff.net - - - kcooney - Kevin Cooney - kcooney@google.com - - - stefanbirkner - Stefan Birkner - mail@stefan-birkner.de - - - marcphilipp - Marc Philipp - mail@marcphilipp.de - - - - - JUnit contributors - JUnit - team@junit.org - https://github.com/junit-team/junit4/graphs/contributors - - developers - - - - - - 3.0.4 - - - - scm:git:git://github.com/junit-team/junit4.git - scm:git:git@github.com:junit-team/junit4.git - https://github.com/junit-team/junit4 - r4.13.2 - - - github - https://github.com/junit-team/junit4/issues - - - github - https://github.com/junit-team/junit4/actions - - - https://github.com/junit-team/junit4/wiki/Download-and-Install - - junit-snapshot-repo - Nexus Snapshot Repository - https://oss.sonatype.org/content/repositories/snapshots/ - - - junit-releases-repo - Nexus Release Repository - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - junit.github.io - gitsite:git@github.com/junit-team/junit4.git - - - - - 1.5 - 2.19.1 - 1.3 - 1.4 - 2.6 - 2.10.3 - ISO-8859-1 - - 67893CC4 - - - - - org.hamcrest - hamcrest-core - ${hamcrestVersion} - - - - org.hamcrest - hamcrest-library - ${hamcrestVersion} - test - - - - - - - ${project.basedir}/src/main/resources - - - ${project.basedir} - - LICENSE-junit.txt - - - - - - - - maven-enforcer-plugin - ${enforcerPluginVersion} - - - enforce-versions - initialize - - enforce - - - true - - - - Current version of Maven ${maven.version} required to build the project - should be ${project.prerequisites.maven}, or higher! - - [${project.prerequisites.maven},) - - - Current JDK version ${java.version} should be ${jdkVersion}, or higher! - - ${jdkVersion} - - - Best Practice is to never define repositories in pom.xml (use a repository - manager instead). - - - - No Snapshots Dependencies Allowed! - - - - - - - - - com.google.code.maven-replacer-plugin - replacer - 1.5.3 - - - process-sources - - replace - - - - - false - ${project.build.sourceDirectory}/junit/runner/Version.java.template - ${project.build.sourceDirectory}/junit/runner/Version.java - false - @version@ - ${project.version} - - - - - maven-compiler-plugin - 3.3 - - ${project.build.sourceEncoding} - ${jdkVersion} - ${jdkVersion} - ${jdkVersion} - ${jdkVersion} - 1.5 - true - true - true - true - - -Xlint:unchecked - - 128m - - - - org.codehaus.mojo - animal-sniffer-maven-plugin - 1.14 - - - signature-check - test - - check - - - - org.codehaus.mojo.signature - java15 - 1.0 - - - - - - - - maven-surefire-plugin - ${surefireVersion} - - org/junit/tests/AllTests.java - true - false - - - - org.apache.maven.surefire - surefire-junit47 - ${surefireVersion} - - - - - - maven-source-plugin - 2.4 - - - - maven-javadoc-plugin - ${javadocPluginVersion} - - ${basedir}/src/main/javadoc/stylesheet.css - protected - false - false - false - true - true - true - JUnit API - UTF-8 - en - ${jdkVersion} - - - api_${jdkVersion} - http://docs.oracle.com/javase/${jdkVersion}.0/docs/api/ - - - *.internal.* - true - 32m - 128m - true - true - - org.hamcrest:hamcrest-core:* - - - - - maven-release-plugin - 2.5.2 - - forked-path - false - -Pgenerate-docs,junit-release ${arguments} - r@{project.version} - - - - maven-site-plugin - 3.4 - - - com.github.stephenc.wagon - wagon-gitsite - 0.4.1 - - - org.apache.maven.doxia - doxia-module-markdown - 1.5 - - - - - maven-jar-plugin - ${jarPluginVersion} - - - false - - true - - - junit - - - - - - maven-clean-plugin - 2.6.1 - - - maven-deploy-plugin - 2.8.2 - - - maven-install-plugin - 2.5.2 - - - maven-resources-plugin - 2.7 - - - - - - - - maven-project-info-reports-plugin - 2.8 - - false - - - - - - index - dependency-info - modules - license - project-team - scm - issue-tracking - mailing-list - dependency-management - dependencies - dependency-convergence - cim - distribution-management - - - - - - maven-javadoc-plugin - ${javadocPluginVersion} - - javadoc/latest - ${basedir}/src/main/javadoc/stylesheet.css - protected - false - false - false - true - true - true - JUnit API - UTF-8 - en - ${jdkVersion} - - - api_${jdkVersion} - http://docs.oracle.com/javase/${jdkVersion}.0/docs/api/ - - - junit.*,*.internal.* - true - 32m - 128m - true - true - - org.hamcrest:hamcrest-core:* - - - - - - javadoc - - - - - - - - - - junit-release - - - - - - maven-gpg-plugin - 1.6 - - - gpg-sign - verify - - sign - - - - - - - - - generate-docs - - - - - maven-source-plugin - - - attach-sources - prepare-package - - jar-no-fork - - - - - - maven-javadoc-plugin - - - attach-javadoc - package - - jar - - - - - - - - - restrict-doclint - - - [1.8,) - - - - - maven-compiler-plugin - - - -Xlint:unchecked - -Xdoclint:accessibility,reference,syntax - - - - - maven-javadoc-plugin - - -Xdoclint:accessibility -Xdoclint:reference - - - - - - - - maven-javadoc-plugin - - -Xdoclint:accessibility -Xdoclint:reference - - - - - - - java9 - - [1.9,12) - - - - 1.6 - - - - - maven-javadoc-plugin - - 1.6 - - - - - - - - maven-javadoc-plugin - - 1.6 - - - - - - - java12 - - [12,) - - - - 1.7 - 3.0.0-M3 - 3.2.0 - 3.2.0 - - - - - maven-javadoc-plugin - - 1.7 - false - - - - maven-compiler-plugin - - - -Xdoclint:none - - - - - - - - - maven-javadoc-plugin - - 1.7 - false - - - - - - - diff --git a/internal/test/integration/maven_test.go b/internal/test/integration/maven_test.go index cd0830e..99dd24a 100644 --- a/internal/test/integration/maven_test.go +++ b/internal/test/integration/maven_test.go @@ -16,16 +16,24 @@ import ( ) const ( - testMavenRepoName = "junit-fixture" - testMavenDistributionBasePath = "junit-fixture" + testMavenRepoName = "foo-fixture" + testMavenDistributionBasePath = "foo-fixture" testMavenRemoteURL = "http://maven_fixture/" testMavenContentOrigin = "http://localhost:8088" - testMavenGroupID = "junit" - testMavenArtifactID = "junit" - testMavenVersion = "4.13.2" - testMavenPomPath = "junit/junit/4.13.2/junit-4.13.2.pom" + testMavenGroupID = "foo" + testMavenArtifactID = "bar" + testMavenVersion123 = "1.2.3" + testMavenVersion201 = "2.0.1" ) +var testMavenArtifactPaths = []string{ + "foo/bar/1.2.3/foo.bar.1.2.3.pom", + "foo/bar/1.2.3/foo.bar.1.2.3.rhlw-00001.jar", + "foo/bar/1.2.3/foo.bar.1.2.3.rhlw-00002.jar", + "foo/bar/2.0.1/foo.bar.2.0.1.pom", + "foo/bar/2.0.1/foo.bar.2.0.1.rhlw-00001.jar", +} + type MavenSuite struct { suite.Suite client *zestwrapper.MavenZest @@ -46,8 +54,10 @@ func (m *MavenSuite) createTestRepository(t *testing.T) { ) require.NoError(t, err) - err = m.client.FetchArtifact(testMavenContentOrigin, m.domainName, testMavenDistributionBasePath, testMavenPomPath) - require.NoError(t, err) + for _, artifactPath := range testMavenArtifactPaths { + err = m.client.FetchArtifact(testMavenContentOrigin, m.domainName, testMavenDistributionBasePath, artifactPath) + require.NoError(t, err) + } addTask, err := m.client.AddCachedContent(repoHref, remoteHref) require.NoError(t, err) @@ -95,21 +105,21 @@ func (m *MavenSuite) TestMavenPackageList() { pkg := response.Results[0] assert.Equal(m.T(), testMavenGroupID, pkg.GroupID) assert.Equal(m.T(), testMavenArtifactID, pkg.ArtifactID) - assert.Contains(m.T(), pkg.Versions, testMavenVersion) + assert.Contains(m.T(), pkg.Versions, testMavenVersion123) + assert.Contains(m.T(), pkg.Versions, testMavenVersion201) require.NotEmpty(m.T(), pkg.LatestReleases) - foundVersion := false + foundVersions := make(map[string]bool) for _, latest := range pkg.LatestReleases { - if latest.Version == testMavenVersion { - foundVersion = true - assert.NotEmpty(m.T(), latest.CreatedAt) - } + foundVersions[latest.Version] = true + assert.NotEmpty(m.T(), latest.CreatedAt) } - assert.True(m.T(), foundVersion) + assert.True(m.T(), foundVersions[testMavenVersion123]) + assert.True(m.T(), foundVersions[testMavenVersion201]) } func (m *MavenSuite) TestMavenPackageListSearchFilter() { - response, err := m.tangy.MavenPackageList(context.Background(), m.repositoryHref, tangy.MavenPackageListFilters{Search: "junit"}, tangy.PageOptions{ + response, err := m.tangy.MavenPackageList(context.Background(), m.repositoryHref, tangy.MavenPackageListFilters{Search: "foo"}, tangy.PageOptions{ Offset: 0, Limit: 10, }) @@ -159,7 +169,7 @@ func (m *MavenSuite) TestMavenBuildList() { m.repositoryHref, testMavenGroupID, testMavenArtifactID, - testMavenVersion, + testMavenVersion123, tangy.PageOptions{Offset: 0, Limit: 10}, ) require.NoError(m.T(), err) @@ -170,7 +180,7 @@ func (m *MavenSuite) TestMavenBuildList() { build := response.Results[0] assert.Equal(m.T(), testMavenGroupID, build.GroupID) assert.Equal(m.T(), testMavenArtifactID, build.ArtifactID) - assert.Equal(m.T(), testMavenVersion, build.Version) + assert.Equal(m.T(), testMavenVersion123, build.Version) assert.True(m.T(), strings.HasSuffix(build.Filename, ".pom"), build.Filename) assert.NotEmpty(m.T(), build.CreatedAt) } @@ -181,7 +191,7 @@ func (m *MavenSuite) TestMavenBuildListPagination() { m.repositoryHref, testMavenGroupID, testMavenArtifactID, - testMavenVersion, + testMavenVersion123, tangy.PageOptions{Offset: 0, Limit: 1}, ) require.NoError(m.T(), err) @@ -193,7 +203,7 @@ func (m *MavenSuite) TestMavenBuildListPagination() { m.repositoryHref, testMavenGroupID, testMavenArtifactID, - testMavenVersion, + testMavenVersion123, tangy.PageOptions{Offset: 1, Limit: 10}, ) require.NoError(m.T(), err) @@ -202,7 +212,7 @@ func (m *MavenSuite) TestMavenBuildListPagination() { } func (m *MavenSuite) TestMavenBuildListEmptyHref() { - response, err := m.tangy.MavenBuildList(context.Background(), "", testMavenGroupID, testMavenArtifactID, testMavenVersion, tangy.PageOptions{Limit: 10}) + response, err := m.tangy.MavenBuildList(context.Background(), "", testMavenGroupID, testMavenArtifactID, testMavenVersion123, tangy.PageOptions{Limit: 10}) require.NoError(m.T(), err) assert.Empty(m.T(), response.Results) assert.Zero(m.T(), response.Total) @@ -219,12 +229,11 @@ func (m *MavenSuite) TestMavenBuildListOptionalFilters() { ) require.NoError(m.T(), err) require.NotEmpty(m.T(), response.Results) - assert.Equal(m.T(), 1, response.Total) + assert.Equal(m.T(), 2, response.Total) build := response.Results[0] assert.Equal(m.T(), testMavenGroupID, build.GroupID) assert.Equal(m.T(), testMavenArtifactID, build.ArtifactID) - assert.Equal(m.T(), testMavenVersion, build.Version) filtered, err := m.tangy.MavenBuildList( context.Background(), @@ -235,8 +244,8 @@ func (m *MavenSuite) TestMavenBuildListOptionalFilters() { tangy.PageOptions{Offset: 0, Limit: 10}, ) require.NoError(m.T(), err) - require.Len(m.T(), filtered.Results, 1) - assert.Equal(m.T(), 1, filtered.Total) + require.Len(m.T(), filtered.Results, 2) + assert.Equal(m.T(), 2, filtered.Total) filtered, err = m.tangy.MavenBuildList( context.Background(), @@ -247,15 +256,16 @@ func (m *MavenSuite) TestMavenBuildListOptionalFilters() { tangy.PageOptions{Offset: 0, Limit: 10}, ) require.NoError(m.T(), err) - require.Len(m.T(), filtered.Results, 1) - assert.Equal(m.T(), 1, filtered.Total) + require.Len(m.T(), filtered.Results, 2) + assert.Equal(m.T(), 2, filtered.Total) } func (m *MavenSuite) TestMavenRepositoryMetrics() { metrics, err := m.tangy.MavenRepositoryMetrics(context.Background(), m.repositoryHref) require.NoError(m.T(), err) assert.Equal(m.T(), 1, metrics.PackageCount) - assert.Equal(m.T(), 1, metrics.BuildCount) + assert.Equal(m.T(), 3, metrics.BuildCount) + assert.Equal(m.T(), 2, metrics.VersionCount) } func (m *MavenSuite) TestMavenRepositoryMetricsEmptyHref() { @@ -263,4 +273,5 @@ func (m *MavenSuite) TestMavenRepositoryMetricsEmptyHref() { require.NoError(m.T(), err) assert.Zero(m.T(), metrics.PackageCount) assert.Zero(m.T(), metrics.BuildCount) + assert.Zero(m.T(), metrics.VersionCount) } diff --git a/internal/test/integration/python_test.go b/internal/test/integration/python_test.go index 576d17c..f6d8aa4 100644 --- a/internal/test/integration/python_test.go +++ b/internal/test/integration/python_test.go @@ -17,6 +17,7 @@ import ( const ( testPythonRepoName = "shelf-reader-fixture" testPythonMultiVersionRepoName = "idna-multi-version-fixture" + testPythonMetricsRepoName = "idna-metrics-fixture" testPythonMultiVersionPackage = "idna" testPythonMultiVersionKeepCount = int64(2) testPythonRepoURL = "https://pypi.org/" @@ -320,10 +321,26 @@ func (p *PythonSuite) TestPythonBuildList() { } func (p *PythonSuite) TestPythonRepositoryMetrics() { - metrics, err := p.tangy.PythonRepositoryMetrics(context.Background(), p.repositoryHref) + repoHref, remoteHref, err := p.client.CreateRepository( + p.domainName, + testPythonMetricsRepoName, + testPythonRepoURL, + []string{testPythonMultiVersionPackage}, + testPythonMultiVersionKeepCount, + ) + require.NoError(p.T(), err) + + syncTask, err := p.client.SyncPythonRepository(repoHref, remoteHref) + require.NoError(p.T(), err) + + _, err = p.client.PollTask(syncTask) + require.NoError(p.T(), err) + + metrics, err := p.tangy.PythonRepositoryMetrics(context.Background(), repoHref) require.NoError(p.T(), err) assert.Equal(p.T(), 1, metrics.PackageCount) - assert.Equal(p.T(), 1, metrics.BuildCount) + assert.Equal(p.T(), 2, metrics.BuildCount) + assert.Equal(p.T(), 2, metrics.VersionCount) } func (p *PythonSuite) TestPythonRepositoryMetricsEmptyHref() { @@ -331,6 +348,7 @@ func (p *PythonSuite) TestPythonRepositoryMetricsEmptyHref() { require.NoError(p.T(), err) assert.Zero(p.T(), metrics.PackageCount) assert.Zero(p.T(), metrics.BuildCount) + assert.Zero(p.T(), metrics.VersionCount) } func (p *PythonSuite) TestPythonPackageGetNotFound() { diff --git a/pkg/tangy/maven.go b/pkg/tangy/maven.go index ee78bac..c433170 100644 --- a/pkg/tangy/maven.go +++ b/pkg/tangy/maven.go @@ -55,6 +55,7 @@ type MavenBuildListResponse struct { type MavenRepositoryMetrics struct { PackageCount int `json:"package_count"` BuildCount int `json:"build_count"` + VersionCount int `json:"version_count"` } type mavenArtifactQueryResult struct { @@ -409,8 +410,8 @@ func (t *tangyImpl) MavenBuildList(ctx context.Context, repositoryHref, groupID, }, nil } -// MavenRepositoryMetrics returns package and build counts for the latest version of a repository. -// Packages are distinct group_id/artifact_id pairs; builds are .pom artifacts. +// MavenRepositoryMetrics returns package, build, and version counts for the latest version of a repository. +// Packages are distinct group_id/artifact_id pairs from .pom artifacts; builds and versions are counted from .jar artifacts. func (t *tangyImpl) MavenRepositoryMetrics(ctx context.Context, repositoryHref string) (MavenRepositoryMetrics, error) { if repositoryHref == "" { return MavenRepositoryMetrics{}, nil @@ -444,19 +445,25 @@ func (t *tangyImpl) MavenRepositoryMetrics(ctx context.Context, repositoryHref s } pomFilter := ` AND rp.filename LIKE '%.pom'` - artifactFrom := ` + jarFilter := ` AND rp.filename LIKE '%.jar'` + pomFrom := ` FROM maven_mavenartifact rp ` + innerUnion + pomFilter + jarFrom := ` + FROM maven_mavenartifact rp + ` + innerUnion + jarFilter metricsQuery := ` SELECT (SELECT COUNT(DISTINCT (rp.group_id, rp.artifact_id)) - ` + artifactFrom + `) AS package_count, + ` + pomFrom + `) AS package_count, (SELECT COUNT(*) - ` + artifactFrom + `) AS build_count` + ` + jarFrom + `) AS build_count, + (SELECT COUNT(DISTINCT (rp.group_id, rp.artifact_id, rp.version)) + ` + jarFrom + `) AS version_count` var metrics MavenRepositoryMetrics - err = conn.QueryRow(ctx, metricsQuery, args).Scan(&metrics.PackageCount, &metrics.BuildCount) + err = conn.QueryRow(ctx, metricsQuery, args).Scan(&metrics.PackageCount, &metrics.BuildCount, &metrics.VersionCount) if err != nil { return MavenRepositoryMetrics{}, err } diff --git a/pkg/tangy/maven_test.go b/pkg/tangy/maven_test.go index 5b6c690..40853bc 100644 --- a/pkg/tangy/maven_test.go +++ b/pkg/tangy/maven_test.go @@ -156,6 +156,7 @@ func TestMockTangyMavenRepositoryMetrics(t *testing.T) { expected := MavenRepositoryMetrics{ PackageCount: 1, BuildCount: 1, + VersionCount: 1, } mockTangy.On("MavenRepositoryMetrics", ctx, repoHref).Return(expected, nil) diff --git a/pkg/tangy/python.go b/pkg/tangy/python.go index 6877764..2ff2d42 100644 --- a/pkg/tangy/python.go +++ b/pkg/tangy/python.go @@ -53,6 +53,7 @@ type PythonBuildListResponse struct { type PythonRepositoryMetrics struct { PackageCount int `json:"package_count"` BuildCount int `json:"build_count"` + VersionCount int `json:"version_count"` } type PythonDistributionListItem struct { @@ -520,7 +521,8 @@ func (t *tangyImpl) PythonBuildList(ctx context.Context, repositoryHref, nameNor }, nil } -// PythonRepositoryMetrics returns package and build counts for the latest version of a repository. +// PythonRepositoryMetrics returns package, build, and version counts for the latest version of a repository. +// Build count equals version count (distinct name_normalized + version pairs). func (t *tangyImpl) PythonRepositoryMetrics(ctx context.Context, repositoryHref string) (PythonRepositoryMetrics, error) { if repositoryHref == "" { return PythonRepositoryMetrics{}, nil @@ -566,14 +568,16 @@ func (t *tangyImpl) PythonRepositoryMetrics(ctx context.Context, repositoryHref SELECT 1 ` + contentFrom + ` GROUP BY rp.name_normalized, rp.version - ) builds) AS build_count` + ) versions) AS version_count` var metrics PythonRepositoryMetrics - err = conn.QueryRow(ctx, metricsQuery, args).Scan(&metrics.PackageCount, &metrics.BuildCount) + err = conn.QueryRow(ctx, metricsQuery, args).Scan(&metrics.PackageCount, &metrics.VersionCount) if err != nil { return PythonRepositoryMetrics{}, err } + metrics.BuildCount = metrics.VersionCount + return metrics, nil } diff --git a/pkg/tangy/python_test.go b/pkg/tangy/python_test.go index 7c66425..d334546 100644 --- a/pkg/tangy/python_test.go +++ b/pkg/tangy/python_test.go @@ -393,6 +393,7 @@ func TestMockTangyPythonRepositoryMetrics(t *testing.T) { expected := PythonRepositoryMetrics{ PackageCount: 10, BuildCount: 25, + VersionCount: 25, } mockTangy.On("PythonRepositoryMetrics", ctx, repoHref).Return(expected, nil)