WIP: 20241113

add.go

@@ -0,0 +1,75 @@
+package altcrawlhqserver
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	"github.com/internetarchive/gocrawlhq"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func addHandler(c *gin.Context) {
+	project := c.Param("project")
+	if !isAuthorized(c) {
+		c.JSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"})
+		return
+	}
+	// identifier := c.GetHeader("X-Identifier")
+
+	addPayload := gocrawlhq.AddPayload{}
+	err := c.BindJSON(&addPayload)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+	fmt.Printf("Discovered: %v\n", addPayload)
+
+	collection := mongoDatabase.Collection(project)
+
+	opts := options.InsertMany().SetOrdered(false)
+	urls := make([]interface{}, len(addPayload.URLs))
+	for i, url := range addPayload.URLs {
+		urls[i] = url
+	}
+	result, err := collection.InsertMany(context.TODO(), urls, opts)
+
+	if addPayload.BypassSeencheck {
+
+	}
+
+	c.JSON(http.StatusCreated, gin.H{"message": "Added"})
+}
+
+// func inspectSeencheck(collection *mongo.Collection, URLs []gocrawlhq.URL, URLType string) ([]gocrawlhq.URL, error) {
+// 	values := []string{}
+// 	for _, URL := range URLs {
+// 		values = append(values, URL.Value)
+// 	}
+
+// 	result, err := collection.Find(
+// 		context.TODO(),
+// 		bson.M{
+// 			"type": URLType,
+// 			"value": bson.M{
+// 				"$in": values,
+// 			},
+// 		},
+// 	)
+// 	if err != nil {
+// 		panic(err)
+// 	}
+// 	defer result.Close(context.Background())
+
+// 	URLs = []gocrawlhq.URL{}
+// 	for result.Next(context.Background()) {
+// 		var URL gocrawlhq.URL
+// 		err := result.Decode(&URL)
+// 		if err != nil {
+// 			panic(err)
+// 		}
+// 		URLs = append(URLs, URL)
+// 	}
+
+// }

altcrawlhq_server.go

@@ -12,31 +12,16 @@ import (
 	"go.mongodb.org/mongo-driver/mongo/options"
 )
 
-func isAuthorized(c *gin.Context) bool {
-	authKey := c.GetHeader("X-Auth-Key")
-	authSecret := c.GetHeader("X-Auth-Secret")
-	// identifier := c.GetHeader("X-Identifier")
-
-	if authKey == "" || authSecret == "" {
-		return false
-	}
-
-	if authKey == "saveweb_key" && authSecret == "saveweb_sec" {
-		return true
-	}
-
-	return false
-}
-
 type FeedRequest struct {
 	Size     int    `json:"size"`
 	Strategy string `json:"strategy"`
 }
 
 var MONGODB_URI string = os.Getenv("MONGODB_URI")
-
 var mongoDatabase *mongo.Database
 
+const MONGODB_DB string = "crawlhq"
+
 func connect_to_mongodb() {
 	fmt.Println("Connecting to MongoDB...")
 	fmt.Println("MONGODB_URI: len=", len(MONGODB_URI))
@@ -55,7 +40,7 @@ func connect_to_mongodb() {
 	}
 	fmt.Println("Connected to MongoDB!")
 
-	db := client.Database("crawlhq")
+	db := client.Database(MONGODB_DB)
 	mongoDatabase = db
 }
 
@@ -65,11 +50,6 @@ func init() {
 
 func ServeHTTP() {
 	g := gin.New()
-	// g.Use(gin.Recovery())
-	// err := g.SetTrustedProxies(nil)
-	// if err != nil {
-	// 	panic(err)
-	// }
 	g.GET("/", func(c *gin.Context) {
 		time.Sleep(1 * time.Second)
 		c.JSON(http.StatusNotFound, gin.H{
@@ -79,11 +59,14 @@ func ServeHTTP() {
 
 	apiGroup := g.Group("/api")
 	{
-		projectGroup := apiGroup.Group("/project/:project/")
+		projectsGroup := apiGroup.Group("/projects/:project/")
 		{
-			projectGroup.GET("/feed", feedHandler)
-			projectGroup.POST("/finished", finishHandler)
-			projectGroup.POST("/discovered", discoveredHandler)
+			projectsGroup.POST("/urls", addHandler)
+			projectsGroup.GET("/urls", feedHandler)
+			projectsGroup.DELETE("/urls", finishHandler)
+
+			projectsGroup.POST("/seencheck", seencheckHandler)
+			projectsGroup.POST("/reset", resetHandler)
 		}
 		apiGroup.GET("/ws", websocketHandler)
 		apiGroup.GET("/online", onlineClientsHandler)

auth.go

@@ -0,0 +1,19 @@
+package altcrawlhqserver
+
+import "github.com/gin-gonic/gin"
+
+func isAuthorized(c *gin.Context) bool {
+	authKey := c.GetHeader("X-Auth-Key")
+	authSecret := c.GetHeader("X-Auth-Secret")
+	// identifier := c.GetHeader("X-Identifier")
+
+	if authKey == "" || authSecret == "" {
+		return false
+	}
+
+	if authKey == "saveweb_key" && authSecret == "saveweb_sec" {
+		return true
+	}
+
+	return false
+}

comment.go

@@ -0,0 +1 @@
+package altcrawlhqserver

feed.go

@@ -17,11 +17,12 @@ import (
 
 // mongodb record
 type URLRecord struct {
-	ID     primitive.ObjectID `json:"id" bson:"_id"`
-	URL    string             `json:"url" bson:"url"`
-	Hop    int                `json:"hop" bson:"hop"`
-	Via    string             `json:"via" bson:"via"`
-	Status string             `json:"status" bson:"status"`
+	gocrawlhq.URL
+	ID primitive.ObjectID `json:"id" bson:"_id"`
+	// URL    string             `json:"url" bson:"url"`
+	Hop    int    `json:"hop" bson:"hop"`
+	Via    string `json:"via" bson:"via"`
+	Status string `json:"status" bson:"status"`
 }
 
 type feedRequest struct {

go.mod

@@ -1,11 +1,11 @@
 module github.com/saveweb/altcrawlhq_server
 
-go 1.22.6
+go 1.23.2
 
 require (
 	github.com/gin-gonic/gin v1.10.0
 	github.com/gorilla/websocket v1.5.3
-	github.com/internetarchive/gocrawlhq v1.2.14
+	github.com/internetarchive/gocrawlhq v1.2.19
 	github.com/jellydator/ttlcache/v3 v3.3.0
 	go.mongodb.org/mongo-driver v1.17.1
 )
@@ -36,6 +36,7 @@ require (
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
+	github.com/upper/db/v4 v4.9.0 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
@@ -44,8 +45,9 @@ require (
 	golang.org/x/crypto v0.26.0 // indirect
 	golang.org/x/net v0.25.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.25.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/text v0.17.0 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )