Incorporate API and analysis capabilities for the Geppetto User Interface

.dockerignore

@@ -1,2 +1,4 @@
 Dockerfile
+.dockerignore
+.git/
 testfiles

Dockerfile

@@ -1,15 +1,19 @@
 FROM golang:1.14-buster as gobuild
 
-WORKDIR /go
+WORKDIR /go/src/github.com/mediaforensics/medifor/
+COPY . ./
 
-ENV GOPATH ""
+RUN apt-get update && apt-get upgrade -y && apt-get install -y protobuf-compiler
+RUN go get -u github.com/golang/protobuf/protoc-gen-go
+RUN cd pkg && ./protoc.sh
 
-COPY . ./
-RUN cd cmd/analyticproxy && go build; cd -; \
-    cd cmd/medifor && go build; cd - \
-    cd cmd/analyticworker && go build; cd -  \
-    cd cmd/analyticworkflow && go build; cd - \
-    cd cmd/fusionworker && go build; cd -
+#ENV GOPATH ""
+
+RUN mkdir -p /app/bin/
+RUN for cmdname in `ls cmd/`; \
+      do \
+        cd cmd/$cmdname && go build && cp $cmdname /app/bin/ && cd -; \
+    done
 
 FROM python:3.8-slim-buster as pybuild
 
@@ -28,11 +32,7 @@ RUN mkdir -p /app/bin \
  && apt-get update \
  && apt-get install -y libmagic1
 
-COPY --from=gobuild /go/cmd/analyticproxy/analyticproxy /app/bin/
-COPY --from=gobuild /go/cmd/medifor/medifor /app/bin/
-COPY --from=gobuild /go/cmd/analyticworker/analyticworker /app/bin
-COPY --from=gobuild /go/cmd/analyticworkflow/analyticworkflow /app/bin
-COPY --from=gobuild /go/cmd/fusionworker/fusionworker /app/bin
+COPY --from=gobuild /app/bin/ /app/bin/
 COPY --from=pybuild /usr/local/lib/python3.8/ /usr/local/lib/python3.8/
 
 ENV PATH ${PATH}:/app/bin

NOTICE

@@ -1,2 +1,16 @@
-This research was developed with funding from the Defense Advanced Research Projects Agency (DARPA).
-Distribution Statement "A" (Approved for Public Release, Distribution Unlimited).
+This tool was originally developed with funding from the Defense Advanced
+Research Projects Agency (DARPA). Distribution Statement "A" (Approved for
+Public Release, Distribution Unlimited).
+
+Modifications to support the Geppetto User Interface are Copyright (2020). Triad
+National Security, LLC. All rights reserved.
+
+This program was produced under U.S. Government contract 89233218CNA000001 for
+Los Alamos National Laboratory (LANL), which is operated by Triad National
+Security, LLC for the U.S. Department of Energy/National Nuclear Security
+Administration. All rights in the program are reserved by Triad National
+Security, LLC, and the U.S. Department of Energy/National Nuclear Security
+Administration. The Government is granted for itself and others acting on its
+behalf a nonexclusive, paid-up, irrevocable worldwide license in this material
+to reproduce, prepare derivative works, distribute copies to the public, perform
+publicly and display publicly, and to permit others to do so.

README.md

@@ -1,5 +1,7 @@
 # MediaForensics Protocol Buffer and Analytic Service Code
 
+This repository extends the [MediaForensics Protocol](https://github.com/mediaforensics/medifor) to support the API and analysis capabilities of the [Geppetto User Interface](https://github.com/lanl/geppetto-ui).
+
 This repository is intended to provide all of the tools necessary to build and test
 media forensic analytics for use as microservices within a distributed system.
 Currently the repository contains the analytic wrapper which is used to provide a

cmd/analyticworkflow/cmd/analytics.go

@@ -0,0 +1,166 @@
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+
+	sq "github.com/Masterminds/squirrel"
+	"github.com/pkg/errors"
+
+	pb "github.com/mediaforensics/medifor/pkg/mediforproto"
+)
+
+func (s *apiService) GetAnalyticsWithScores(ctx context.Context, req *pb.GetAnalyticsWithScoresRequest) (*pb.GetAnalyticsWithScoresResponse, error) {
+	analyticIds, err := s.getAnalyticsWithScores(ctx, req)
+
+	if err != nil {
+		return nil, errors.Wrap(err, "analytics with scores")
+	}
+
+	fuserIds, err := s.getFusersWithScores(ctx, req)
+
+	if err != nil {
+		return nil, errors.Wrap(err, "analytics with scores")
+	}
+
+	resp := &pb.GetAnalyticsWithScoresResponse{
+		AnalyticIds: analyticIds,
+		FuserIds:    fuserIds,
+	}
+
+	return resp, nil
+}
+
+func (s *apiService) getAnalyticsWithScores(ctx context.Context, req *pb.GetAnalyticsWithScoresRequest) ([]string, error) {
+	analyticIds := []string{}
+
+	query := sq.Select("analytic_id").
+		Distinct().
+		From("detection").
+		LeftJoin("detectionTag USING (detection_id)").
+		Where(sq.NotEq{"finished": nil}).
+		Where("coalesce(detection->'status'->>'code', '0') = '0'").
+		Where(sq.Or{
+			sq.And{
+				sq.Expr("detection ?? 'imgManip'"),
+				sq.Expr("coalesce(detection->'imgManip'->>'optOut', 'OPT_OUT_NONE') NOT IN ('OPT_OUT_ALL', 'OPT_OUT_DETECTION')"),
+			},
+			sq.And{
+				sq.Expr("detection ?? 'vidManip'"),
+				sq.Expr("NOT (coalesce(detection->'vidManip'->'optOut', '[]'::jsonb) ?? 'VIDEO_OPT_OUT_DETECTION')"),
+			},
+		})
+
+	if req.Tags != nil {
+		tags, err := json.Marshal(req.Tags)
+
+		if err != nil {
+			return nil, err
+		}
+
+		query = query.Where("(tags || user_tags) @> ?", tags)
+	}
+
+	if req.ExcludeTags != nil {
+		excludeTags, err := json.Marshal(req.ExcludeTags)
+
+		if err != nil {
+			return nil, err
+		}
+
+		query = query.Where("NOT (user_tags ??| array(SELECT jsonb_object_keys(?)))", excludeTags)
+	}
+
+	rows, err := query.PlaceholderFormat(sq.Dollar).
+		RunWith(s.pgdb).
+		QueryContext(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+
+	defer rows.Close()
+
+	for rows.Next() {
+		var analyticId string
+
+		if err := rows.Scan(&analyticId); err != nil {
+			return nil, err
+		}
+
+		analyticIds = append(analyticIds, analyticId)
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	return analyticIds, nil
+}
+
+func (s *apiService) getFusersWithScores(ctx context.Context, req *pb.GetAnalyticsWithScoresRequest) ([]string, error) {
+	fuserIds := []string{}
+
+	query := sq.Select("fuser_id").
+		Distinct().
+		From("fusion").
+		LeftJoin("detectionTag USING (detection_id)").
+		Where(sq.NotEq{"finished": nil}).
+		Where("coalesce(fusion->'status'->>'code', '0') = '0'").
+		Where(sq.Or{
+			sq.And{
+				sq.Expr("fusion ?? 'imgManip'"),
+				sq.Expr("coalesce(fusion->'imgManip'->>'optOut', 'OPT_OUT_NONE') NOT IN ('OPT_OUT_ALL', 'OPT_OUT_DETECTION')"),
+			},
+			sq.And{
+				sq.Expr("fusion ?? 'vidManip'"),
+				sq.Expr("NOT (coalesce(fusion->'vidManip'->'optOut', '[]'::jsonb) ?? 'VIDEO_OPT_OUT_DETECTION')"),
+			},
+		})
+
+	if req.Tags != nil {
+		tags, err := json.Marshal(req.Tags)
+
+		if err != nil {
+			return nil, err
+		}
+
+		query = query.Where("(tags || user_tags) @> ?", tags)
+	}
+
+	if req.ExcludeTags != nil {
+		excludeTags, err := json.Marshal(req.ExcludeTags)
+
+		if err != nil {
+			return nil, err
+		}
+
+		query = query.Where("NOT ((tags || user_tags) @> ?)", excludeTags)
+	}
+
+	rows, err := query.PlaceholderFormat(sq.Dollar).
+		RunWith(s.pgdb).
+		QueryContext(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+
+	defer rows.Close()
+
+	for rows.Next() {
+		var fuserId string
+
+		if err := rows.Scan(&fuserId); err != nil {
+			return nil, err
+		}
+
+		fuserIds = append(fuserIds, fuserId)
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	return fuserIds, nil
+}