Add support for serving observation properties. (#1426)

datacommonsorg · Oct 4, 2024 · 6b01171 · 6b01171
1 parent 2c94f88
commit 6b01171
Show file tree

Hide file tree

Showing 21 changed files with 670 additions and 265 deletions.
diff --git a/cmd/main.go b/cmd/main.go
@@ -212,6 +212,11 @@ func main() {
 		if err := sqldb.CreateTables(sqlClient); err != nil {
 			log.Fatalf("Can not create tables in database: %v", err)
 		}
+
+		err = sqldb.CheckSchema(sqlClient)
+		if err != nil {
+			log.Fatalf("SQL schema check failed: %v", err)
+		}
 	}
 
 	// Store

diff --git a/internal/server/v2/observation/contained_in.go b/internal/server/v2/observation/contained_in.go
@@ -30,6 +30,7 @@ import (
 	"github.com/datacommonsorg/mixer/internal/server/resource"
 	"github.com/datacommonsorg/mixer/internal/server/stat"
 	"github.com/datacommonsorg/mixer/internal/server/v2/shared"
+	"github.com/datacommonsorg/mixer/internal/sqldb/sqlquery"
 	"github.com/datacommonsorg/mixer/internal/store/bigtable"
 	"github.com/datacommonsorg/mixer/internal/util"
 	"google.golang.org/grpc/codes"
@@ -220,7 +221,7 @@ func FetchContainedIn(
 					return nil, err
 				}
 			}
-			directResp, err := FetchDirectSQL(
+			directResp, err := sqlquery.GetObservations(
 				ctx,
 				store.SQLClient,
 				sqlProvenances,

diff --git a/internal/server/v2/observation/direct.go b/internal/server/v2/observation/direct.go
@@ -17,8 +17,6 @@ package observation
 
 import (
 	"context"
-	"database/sql"
-	"fmt"
 	"net/url"
 	"sort"
 	"strings"
@@ -29,6 +27,7 @@ import (
 	"github.com/datacommonsorg/mixer/internal/server/ranking"
 	"github.com/datacommonsorg/mixer/internal/server/stat"
 	"github.com/datacommonsorg/mixer/internal/server/v2/shared"
+	"github.com/datacommonsorg/mixer/internal/sqldb/sqlquery"
 	"github.com/datacommonsorg/mixer/internal/store"
 	"github.com/datacommonsorg/mixer/internal/store/bigtable"
 	"github.com/datacommonsorg/mixer/internal/util"
@@ -88,7 +87,7 @@ func FetchDirect(
 	if err != nil {
 		return nil, err
 	}
-	sqlObservation, err := FetchDirectSQL(
+	sqlObservation, err := sqlquery.GetObservations(
 		ctx,
 		store.SQLClient,
 		sqlProvenances,
@@ -191,47 +190,3 @@ func FetchDirectBT(
 	}
 	return result, nil
 }
-
-// FetchDirectSQL fetches data from SQLite database.
-func FetchDirectSQL(
-	ctx context.Context,
-	sqlClient *sql.DB,
-	sqlProvenances map[string]*pb.Facet,
-	variables []string,
-	entities []string,
-	queryDate string,
-	filter *pbv2.FacetFilter,
-) (*pbv2.ObservationResponse, error) {
-	result := initObservationResult(variables)
-	if sqlClient == nil {
-		return result, nil
-	}
-	// Construct query
-	entitiesStr := "'" + strings.Join(entities, "', '") + "'"
-	variablesStr := "'" + strings.Join(variables, "', '") + "'"
-	query := fmt.Sprintf(
-		`
-			SELECT entity, variable, date, value, provenance FROM observations
-			WHERE entity IN (%s)
-			AND variable IN (%s)
-			AND value != ''
-		`,
-		entitiesStr,
-		variablesStr,
-	)
-	if queryDate != "" && queryDate != shared.LATEST {
-		query += fmt.Sprintf("AND date = (%s) ", queryDate)
-	}
-	query += "ORDER BY date ASC;"
-	// Execute query
-	rows, err := sqlClient.Query(query)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	tmp, err := handleSQLRows(rows, variables)
-	if err != nil {
-		return nil, err
-	}
-	return processSqlData(result, tmp, queryDate, sqlProvenances), nil
-}
diff --git a/internal/server/v2/observation/golden/contained_in_2015/US_State.json b/internal/server/v2/observation/golden/contained_in_2015/US_State.json
@@ -5,7 +5,7 @@
         "geoId/06": {
           "ordered_facets": [
             {
-              "facet_id": "custom",
+              "facet_id": "2921750529",
               "observations": [
                 {
                   "date": "2015",
@@ -22,7 +22,7 @@
     }
   },
   "facets": {
-    "custom": {
+    "2921750529": {
       "import_name": "Custom Prov",
       "provenance_url": "custom.datacommons.org"
     }