Merge pull request #70 from dacharyc/DOCSP-53367

dacharyc · web-flow · commit e5a8603ae398 · 2025-09-02T17:18:53.000-04:00
(DOCSP-53367): Update version handling for Snooty Data API fetch documents endpoint
diff --git a/audit/gdcd/HandleCollectionSummariesDocument.go b/audit/gdcd/HandleCollectionSummariesDocument.go
@@ -40,13 +40,13 @@ func HandleCollectionSummariesDocument(project types.ProjectDetails, report type
 			}
 		}
 	}
-	if project.ActiveBranch != collectionVersionKey {
+	if project.Version != collectionVersionKey {
 		// If the active branch doesn't match the most recent version, need to make a new CollectionInfoView for this document
 		updatedSummaryDoc := db.MakeNewCollectionVersionDocument(*summaryDoc, project, report)
 		summaryDoc = &updatedSummaryDoc
 	} else {
 		// If the active branch does match the most recent version, just need to update this version document's last updated date and counts
-		pageCountBeforeUpdating := summaryDoc.Version[project.ActiveBranch].TotalPageCount
+		pageCountBeforeUpdating := summaryDoc.Version[project.Version].TotalPageCount
 		updatedSummaryDoc := db.UpdateCollectionVersionDocument(*summaryDoc, project, report)
 		summaryDoc = &updatedSummaryDoc
 
diff --git a/audit/gdcd/UpdateExistingPage.go b/audit/gdcd/UpdateExistingPage.go
@@ -3,8 +3,8 @@ package main
 import (
 	"common"
 	"context"
-	"gdcd/add-code-examples"
-	"gdcd/compare-code-examples"
+	add_code_examples "gdcd/add-code-examples"
+	compare_code_examples "gdcd/compare-code-examples"
 	"gdcd/db"
 	"gdcd/snooty"
 	"gdcd/types"
diff --git a/audit/gdcd/UpdateProjectReportForNewPage.go b/audit/gdcd/UpdateProjectReportForNewPage.go
@@ -2,7 +2,7 @@ package main
 
 import (
 	"common"
-	"gdcd/add-code-examples"
+	add_code_examples "gdcd/add-code-examples"
 	compare_code_examples "gdcd/compare-code-examples"
 	"gdcd/types"
 	"gdcd/utils"
diff --git a/audit/gdcd/add-code-examples/CategorizeDriverLanguageSnippet.go b/audit/gdcd/add-code-examples/CategorizeDriverLanguageSnippet.go
@@ -4,6 +4,7 @@ import (
 	"common"
 	"context"
 	"fmt"
+
 	"github.com/tmc/langchaingo/llms"
 	"github.com/tmc/langchaingo/llms/ollama"
 	"github.com/tmc/langchaingo/prompts"
diff --git a/audit/gdcd/add-code-examples/CategorizeJsonLikeSnippet.go b/audit/gdcd/add-code-examples/CategorizeJsonLikeSnippet.go
@@ -4,6 +4,7 @@ import (
 	"common"
 	"context"
 	"fmt"
+
 	"github.com/tmc/langchaingo/llms"
 	"github.com/tmc/langchaingo/llms/ollama"
 	"github.com/tmc/langchaingo/prompts"
diff --git a/audit/gdcd/add-code-examples/CategorizeShellSnippet.go b/audit/gdcd/add-code-examples/CategorizeShellSnippet.go
@@ -4,6 +4,7 @@ import (
 	"common"
 	"context"
 	"fmt"
+
 	"github.com/tmc/langchaingo/llms"
 	"github.com/tmc/langchaingo/llms/ollama"
 	"github.com/tmc/langchaingo/prompts"
diff --git a/audit/gdcd/add-code-examples/CategorizeTextSnippet.go b/audit/gdcd/add-code-examples/CategorizeTextSnippet.go
@@ -4,6 +4,7 @@ import (
 	"common"
 	"context"
 	"fmt"
+
 	"github.com/tmc/langchaingo/llms"
 	"github.com/tmc/langchaingo/llms/ollama"
 	"github.com/tmc/langchaingo/prompts"
diff --git a/audit/gdcd/compare-code-examples/CompareExistingIncomingCodeExampleSlices.go b/audit/gdcd/compare-code-examples/CompareExistingIncomingCodeExampleSlices.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"gdcd/snooty"
 	"gdcd/types"
+
 	"github.com/tmc/langchaingo/llms/ollama"
 )
 
diff --git a/audit/gdcd/compare-code-examples/CompareExistingIncomingCodeExampleSlices_test.go b/audit/gdcd/compare-code-examples/CompareExistingIncomingCodeExampleSlices_test.go
@@ -4,12 +4,13 @@ import (
 	"common"
 	"context"
 	"fmt"
-	"gdcd/add-code-examples"
+	add_code_examples "gdcd/add-code-examples"
 	"gdcd/compare-code-examples/data"
 	"gdcd/types"
-	"github.com/tmc/langchaingo/llms/ollama"
 	"log"
 	"testing"
+
+	"github.com/tmc/langchaingo/llms/ollama"
 )
 
 func TestOneNewCodeExampleHandledCorrectly(t *testing.T) {
diff --git a/audit/gdcd/compare-code-examples/HandleNewPageNodes_test.go b/audit/gdcd/compare-code-examples/HandleNewPageNodes_test.go
@@ -5,9 +5,10 @@ import (
 	add_code_examples "gdcd/add-code-examples"
 	"gdcd/compare-code-examples/data"
 	"gdcd/types"
-	"github.com/tmc/langchaingo/llms/ollama"
 	"log"
 	"testing"
+
+	"github.com/tmc/langchaingo/llms/ollama"
 )
 
 // NOTE: For these tests, I'm just confirming that we're creating the correct number of new code nodes. We don't need to
diff --git a/audit/gdcd/db/MakeNewCollectionVersionDocument.go b/audit/gdcd/db/MakeNewCollectionVersionDocument.go
@@ -12,6 +12,6 @@ func MakeNewCollectionVersionDocument(existingSummaries common.CollectionReport,
 		TotalCodeCount:   report.Counter.IncomingCodeNodesCount,
 		LastUpdatedAtUTC: time.Now().UTC(),
 	}
-	existingSummaries.Version[project.ActiveBranch] = collectionInfo
+	existingSummaries.Version[project.Version] = collectionInfo
 	return existingSummaries
 }
diff --git a/audit/gdcd/db/MakeSummariesDocument.go b/audit/gdcd/db/MakeSummariesDocument.go
@@ -13,7 +13,7 @@ func MakeSummariesDocument(project types.ProjectDetails, report types.ProjectRep
 		LastUpdatedAtUTC: time.Now().UTC(),
 	}
 	versionMap := make(map[string]common.CollectionInfoView)
-	versionMap[project.ActiveBranch] = collectionInfo
+	versionMap[project.Version] = collectionInfo
 	collectionReport := common.CollectionReport{
 		ID:      "summaries",
 		Version: versionMap,
diff --git a/audit/gdcd/db/UpdateCollectionVersionDocument.go b/audit/gdcd/db/UpdateCollectionVersionDocument.go
@@ -7,10 +7,10 @@ import (
 )
 
 func UpdateCollectionVersionDocument(existingSummaries common.CollectionReport, project types.ProjectDetails, report types.ProjectReport) common.CollectionReport {
-	existingCollectionInfo := existingSummaries.Version[project.ActiveBranch]
+	existingCollectionInfo := existingSummaries.Version[project.Version]
 	existingCollectionInfo.TotalPageCount = report.Counter.TotalCurrentPageCount
 	existingCollectionInfo.TotalCodeCount = report.Counter.IncomingCodeNodesCount
 	existingCollectionInfo.LastUpdatedAtUTC = time.Now().UTC()
-	existingSummaries.Version[project.ActiveBranch] = existingCollectionInfo
+	existingSummaries.Version[project.Version] = existingCollectionInfo
 	return existingSummaries
 }
diff --git a/audit/gdcd/main.go b/audit/gdcd/main.go
@@ -66,17 +66,17 @@ func main() {
 	// Uncomment to parse a single project during testing
 	// compass := types.ProjectDetails{
 	//	ProjectName:  "compass",
-	//	ActiveBranch: "master",
+	//	Version: "master",
 	//	ProdUrl:      "https://mongodb.com/docs/compass/current",
 	// }
 	// opsManager := types.ProjectDetails{
 	//	ProjectName:  "ops-manager",
-	//	ActiveBranch: "v8.0",
+	//	Version: "v8.0",
 	//	ProdUrl:      "https://mongodb.com/docs/ops-manager/current",
 	// }
 	// cloudManager := types.ProjectDetails{
 	//	ProjectName:  "cloud-manager",
-	//	ActiveBranch: "master",
+	//	Version: "master",
 	//	ProdUrl:      "https://mongodb.com/docs/cloud-manager/",
 	// }
 	// projectsToParse := []types.ProjectDetails{compass}
diff --git a/audit/gdcd/snooty/GetLangForIoCodeBlock_test.go b/audit/gdcd/snooty/GetLangForIoCodeBlock_test.go
@@ -2,7 +2,6 @@ package snooty
 
 import (
 	"common"
-	test_data "gdcd/snooty/test-data"
 	"gdcd/types"
 	"testing"
 )
@@ -16,14 +15,14 @@ func TestGetLangForIoCodeBlock(t *testing.T) {
 		args args
 		want string
 	}{
-		{"Handles iocodeblock with input lang", args{test_data.MakeIoCodeBlockForTesting(true, true, common.C, true, true, true, false, false)}, common.C},
-		{"Handles iocodeblock with no input lang using child code node lang", args{test_data.MakeIoCodeBlockForTesting(false, true, common.C, true, true, true, false, false)}, common.C},
-		{"Handles iocodeblock with no input lang and no child code node lang using filepath", args{test_data.MakeIoCodeBlockForTesting(false, false, common.C, true, true, true, false, false)}, common.C},
-		{"Lang should be undefined if no other conditions provide lang", args{test_data.MakeIoCodeBlockForTesting(false, false, common.C, false, true, true, false, false)}, common.Undefined},
-		{"Handles no input directive", args{test_data.MakeIoCodeBlockForTesting(true, true, common.C, true, false, true, false, false)}, common.Undefined},
-		{"Handles no child code node directive", args{test_data.MakeIoCodeBlockForTesting(false, true, common.C, true, true, false, false, false)}, common.C},
-		{"Handles input directive not in first position", args{test_data.MakeIoCodeBlockForTesting(true, true, common.C, true, true, true, true, false)}, common.C},
-		{"Handles child code node directive not in first position", args{test_data.MakeIoCodeBlockForTesting(false, true, common.C, true, true, true, false, true)}, common.C},
+		{"Handles iocodeblock with input lang", args{MakeIoCodeBlockForTesting(true, true, common.C, true, true, true, false, false)}, common.C},
+		{"Handles iocodeblock with no input lang using child code node lang", args{MakeIoCodeBlockForTesting(false, true, common.C, true, true, true, false, false)}, common.C},
+		{"Handles iocodeblock with no input lang and no child code node lang using filepath", args{MakeIoCodeBlockForTesting(false, false, common.C, true, true, true, false, false)}, common.C},
+		{"Lang should be undefined if no other conditions provide lang", args{MakeIoCodeBlockForTesting(false, false, common.C, false, true, true, false, false)}, common.Undefined},
+		{"Handles no input directive", args{MakeIoCodeBlockForTesting(true, true, common.C, true, false, true, false, false)}, common.Undefined},
+		{"Handles no child code node directive", args{MakeIoCodeBlockForTesting(false, true, common.C, true, true, false, false, false)}, common.C},
+		{"Handles input directive not in first position", args{MakeIoCodeBlockForTesting(true, true, common.C, true, true, true, true, false)}, common.C},
+		{"Handles child code node directive not in first position", args{MakeIoCodeBlockForTesting(false, true, common.C, true, true, true, false, true)}, common.C},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
diff --git a/audit/gdcd/snooty/GetLangForLiteralInclude_test.go b/audit/gdcd/snooty/GetLangForLiteralInclude_test.go
@@ -2,7 +2,6 @@ package snooty
 
 import (
 	"common"
-	test_data "gdcd/snooty/test-data"
 	"gdcd/types"
 	"testing"
 )
@@ -16,10 +15,10 @@ func TestGetLangForLiteralInclude(t *testing.T) {
 		args args
 		want string
 	}{
-		{"Handles literalinclude with specified lang", args{test_data.MakeLiteralIncludeNodeForTesting(true, common.C, false)}, common.C},
-		{"Handles literalinclude with empty string lang using filepath", args{test_data.MakeLiteralIncludeNodeForTesting(false, common.C, true)}, common.C},
-		{"Uses child code node lang when literalinclude empty string lang and no filepath", args{test_data.MakeLiteralIncludeNodeForTesting(false, common.C, false)}, common.C},
-		{"Lang should be undefined if no other conditions provide lang", args{test_data.MakeLiteralIncludeNodeForTesting(true, common.Undefined, false)}, common.Undefined},
+		{"Handles literalinclude with specified lang", args{MakeLiteralIncludeNodeForTesting(true, common.C, false)}, common.C},
+		{"Handles literalinclude with empty string lang using filepath", args{MakeLiteralIncludeNodeForTesting(false, common.C, true)}, common.C},
+		{"Uses child code node lang when literalinclude empty string lang and no filepath", args{MakeLiteralIncludeNodeForTesting(false, common.C, false)}, common.C},
+		{"Lang should be undefined if no other conditions provide lang", args{MakeLiteralIncludeNodeForTesting(true, common.Undefined, false)}, common.Undefined},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
diff --git a/audit/gdcd/snooty/GetProjectDocuments.go b/audit/gdcd/snooty/GetProjectDocuments.go
@@ -16,7 +16,7 @@ import (
 // for further processing.
 func GetProjectPages(project types.ProjectDetails, client *http.Client) []types.PageWrapper {
 	env := os.Getenv("APP_ENV")
-	apiURL := fmt.Sprintf("https://snooty-data-api.mongodb.com/prod/projects/%s/%s/documents", project.ProjectName, project.ActiveBranch)
+	apiURL := fmt.Sprintf("https://snooty-data-api.mongodb.com/prod/projects/%s/%s/documents", project.ProjectName, project.Version)
 	var reader bufio.Reader
 
 	if env == "testing" {
diff --git a/audit/gdcd/snooty/GetProjectDocuments_test.go b/audit/gdcd/snooty/GetProjectDocuments_test.go
@@ -10,9 +10,9 @@ import (
 // TODO: Figure out why this test is failing. The stub has 13 JSON blobs where "type":"page" - I'm getting one too few back. The first "page" response is always nil. This is not a problem for the C driver.
 func TestSparkConnectorStubShouldReturnPages(t *testing.T) {
 	testProject := types.ProjectDetails{
-		ProjectName:  "spark-connector",
-		ActiveBranch: "",
-		ProdUrl:      "",
+		ProjectName: "spark-connector",
+		Version:     "",
+		ProdUrl:     "",
 	}
 	projectDocuments := GetProjectPages(testProject, &http.Client{Timeout: 5 * time.Second})
 	projectDocumentCount := len(projectDocuments)
@@ -24,9 +24,9 @@ func TestSparkConnectorStubShouldReturnPages(t *testing.T) {
 
 func TestCDriverStubShouldReturnPages(t *testing.T) {
 	testProject := types.ProjectDetails{
-		ProjectName:  "c",
-		ActiveBranch: "",
-		ProdUrl:      "",
+		ProjectName: "c",
+		Version:     "",
+		ProdUrl:     "",
 	}
 	projectDocuments := GetProjectPages(testProject, &http.Client{Timeout: 5 * time.Second})
 	projectDocumentCount := len(projectDocuments)
diff --git a/audit/gdcd/snooty/GetProjects.go b/audit/gdcd/snooty/GetProjects.go
@@ -6,7 +6,9 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
+	"path"
 	"strings"
 )
 
@@ -26,6 +28,16 @@ func removeTrailingSlash(input string) string {
 	return input
 }
 
+func getLastSegment(rawURL string) string {
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		log.Printf("ERROR: failed to parse URL: %s\n", err)
+	}
+	u.Path = strings.TrimSuffix(u.Path, "/")
+	seg := path.Base(u.Path)
+	return seg
+}
+
 func GetProjects(client *http.Client) []types.ProjectDetails {
 	env := os.Getenv("APP_ENV")
 	var response types.Response
@@ -78,7 +90,7 @@ func GetProjects(client *http.Client) []types.ProjectDetails {
 
 	var collectionsToParse []types.ProjectDetails
 	for _, docsProject := range response.Data {
-		var activeBranch string
+		var version string
 		var prodUrl string
 		if !contains(ignoreProjectNames, docsProject.Project) {
 			for _, branch := range docsProject.Branches {
@@ -87,17 +99,20 @@ func GetProjects(client *http.Client) []types.ProjectDetails {
 					// the PageUrl, we add a slash, so we need to remove a trailing slash if one exists here so we don't
 					// have double slashes.
 					urlWithNoTrailingSlash := removeTrailingSlash(branch.FullUrl)
-					activeBranch = branch.GitBranchName
+
+					// A change in Snooty Data API behavior means the branch name from the projects endpoint is no longer
+					// used to fetch the documents for that project. Instead, we should use the last segment of the URL.
+					version = getLastSegment(urlWithNoTrailingSlash)
 					prodUrl = urlWithNoTrailingSlash
 					break
 				}
 			}
 			// If the project does not have an active, stable branch, we don't want to try to get the project details
-			if activeBranch != "" {
+			if version != "" {
 				collectionDetails := types.ProjectDetails{
-					ProjectName:  docsProject.Project,
-					ActiveBranch: activeBranch,
-					ProdUrl:      prodUrl,
+					ProjectName: docsProject.Project,
+					Version:     version,
+					ProdUrl:     prodUrl,
 				}
 				collectionsToParse = append(collectionsToParse, collectionDetails)
 			} else {
diff --git a/audit/gdcd/snooty/GetProjects_test.go b/audit/gdcd/snooty/GetProjects_test.go
@@ -20,11 +20,20 @@ func TestStubbedProjectsReturnTheCorrectNumberOfProjects(t *testing.T) {
 func TestStubbedProjectsReturnCorrectProjectDetails(t *testing.T) {
 	projectDocuments := GetProjects(&http.Client{Timeout: 5 * time.Second})
 	expectedProjectDocument := types.ProjectDetails{
-		ProjectName:  "spark-connector",
-		ActiveBranch: "v10.4",
-		ProdUrl:      "https://mongodb.com/docs/spark-connector/current",
+		ProjectName: "spark-connector",
+		Version:     "v10.4",
+		ProdUrl:     "https://mongodb.com/docs/spark-connector/current",
 	}
 	if !reflect.DeepEqual(projectDocuments[0], expectedProjectDocument) {
 		t.Errorf("FAILED: got %v, want %v", projectDocuments, expectedProjectDocument)
 	}
 }
+
+func TestUrlHandlingToGetDocsProject(t *testing.T) {
+	fullUrl := "https://docs.mongodb.com/drivers/go/current/"
+	docsBranch := getLastSegment(fullUrl)
+	expectedDocsBranch := "current"
+	if docsBranch != expectedDocsBranch {
+		t.Errorf("FAILED: got %s, want %s", docsBranch, expectedDocsBranch)
+	}
+}
diff --git a/audit/gdcd/snooty/MakeCodeNodeForTesting.go b/audit/gdcd/snooty/MakeCodeNodeForTesting.go
@@ -1,16 +1,15 @@
-package test_data
+package snooty
 
 import (
 	"common"
 	add_code_examples "gdcd/add-code-examples"
-	"gdcd/snooty"
 	"time"
 )
 
 func MakeCodeNodeForTesting(language string, category string) common.CodeNode {
 	code := "Some code goes here"
 	fileExtension := add_code_examples.GetFileExtensionFromStringLang(language)
-	sha256Hash := snooty.MakeSha256HashForCode(code)
+	sha256Hash := MakeSha256HashForCode(code)
 	return common.CodeNode{
 		Code:           code,
 		Language:       language,
diff --git a/audit/gdcd/snooty/MakeIoCodeBlockForTesting.go b/audit/gdcd/snooty/MakeIoCodeBlockForTesting.go
@@ -1,4 +1,4 @@
-package test_data
+package snooty
 
 import (
 	add_code_examples "gdcd/add-code-examples"
diff --git a/audit/gdcd/snooty/MakeLiteralIncludeNodeForTesting.go b/audit/gdcd/snooty/MakeLiteralIncludeNodeForTesting.go
@@ -1,4 +1,4 @@
-package test_data
+package snooty
 
 import (
 	add_code_examples "gdcd/add-code-examples"
diff --git a/audit/gdcd/types/SnootyProjectList.go b/audit/gdcd/types/SnootyProjectList.go
@@ -30,7 +30,7 @@ type Response struct {
 }
 
 type ProjectDetails struct {
-	ProjectName  string
-	ActiveBranch string
-	ProdUrl      string
+	ProjectName string
+	Version     string
+	ProdUrl     string
 }

Original file line number	Diff line number	Diff line change
`@@ -40,13 +40,13 @@ func HandleCollectionSummariesDocument(project types.ProjectDetails, report type`
`40`	`40`	`}`
`41`	`41`	`}`
`42`	`42`	`}`
`43`		`- if project.ActiveBranch != collectionVersionKey {`
	`43`	`+ if project.Version != collectionVersionKey {`
`44`	`44`	`// If the active branch doesn't match the most recent version, need to make a new CollectionInfoView for this document`
`45`	`45`	`updatedSummaryDoc := db.MakeNewCollectionVersionDocument(*summaryDoc, project, report)`
`46`	`46`	`summaryDoc = &updatedSummaryDoc`
`47`	`47`	`} else {`
`48`	`48`	`// If the active branch does match the most recent version, just need to update this version document's last updated date and counts`
`49`		`- pageCountBeforeUpdating := summaryDoc.Version[project.ActiveBranch].TotalPageCount`
	`49`	`+ pageCountBeforeUpdating := summaryDoc.Version[project.Version].TotalPageCount`
`50`	`50`	`updatedSummaryDoc := db.UpdateCollectionVersionDocument(*summaryDoc, project, report)`
`51`	`51`	`summaryDoc = &updatedSummaryDoc`
`52`	`52`
Original file line number	Diff line number	Diff line change
`@@ -5,6 +5,7 @@ import (`
`5`	`5`	`"context"`
`6`	`6`	`"gdcd/snooty"`
`7`	`7`	`"gdcd/types"`
	`8`	`+`
`8`	`9`	`"github.com/tmc/langchaingo/llms/ollama"`
`9`	`10`	`)`
`10`	`11`