Merge pull request #33 from cid-harvard/feature/partner-countries

New partner country API geolevels

colombia/api_schemas.py

@@ -70,7 +70,7 @@ class Meta:
         fields = ("export_value", "country_id", "product_id", "year")
 
 
-class CountryDepartmentYearSchema(ma.Schema):
+class CountryXYearSchema(ma.Schema):
 
     class Meta:
         fields = ("export_value", "country_id", "location_id", "year")
@@ -150,7 +150,7 @@ class ColombiaMetadataSchema(MetadataSchema):
 country_municipality_product_year = CountryMunicipalityProductYearSchema(many=True)
 country_department_product_year = CountryDepartmentProductYearSchema(many=True)
 
-country_department_year = CountryDepartmentYearSchema(many=True)
+country_x_year = CountryXYearSchema(many=True)
 
 product_year = ProductYearSchema(many=True)
 industry_year = IndustryYearSchema(many=True)

colombia/data/models.py

@@ -118,21 +118,39 @@ class CountryMunicipalityProductYear(CountryXProductYear):
     __tablename__ = "country_municipality_product_year"
 
 
-class CountryDepartmentYear(BaseModel, IDMixin):
+class CountryXYear(BaseModel, IDMixin):
 
-    __tablename__ = "country_department_year"
+    __abstract__ = True
 
-    country_id = db.Column(db.Integer, db.ForeignKey(Location.id))
-    location_id = db.Column(db.Integer, db.ForeignKey(Location.id))
-    year = db.Column(db.Integer)
+    @declared_attr
+    def country_id(cls):
+        return db.Column(db.Integer, db.ForeignKey(Location.id))
+
+    @declared_attr
+    def location_id(cls):
+        return db.Column(db.Integer, db.ForeignKey(Location.id))
 
-    location = db.relationship(Location, foreign_keys=[location_id])
-    country = db.relationship(Location, foreign_keys=[country_id])
+    year = db.Column(db.Integer)
 
     export_value = db.Column(db.BIGINT)
     export_num_plants = db.Column(db.Integer)
 
 
+class CountryCountryYear(CountryXYear):
+
+    __tablename__ = "country_country_year"
+
+
+class CountryDepartmentYear(CountryXYear):
+
+    __tablename__ = "country_department_year"
+
+
+class CountryMSAYear(CountryXYear):
+
+    __tablename__ = "country_msa_year"
+
+
 class DepartmentYear(BaseModel, IDMixin):
 
     __tablename__ = "department_year"

colombia/data/views.py

@@ -5,7 +5,8 @@
                      MunicipalityIndustryYear, ProductYear, IndustryYear,
                      DepartmentYear, Location, CountryMunicipalityProductYear,
                      CountryDepartmentProductYear, OccupationYear,
-                     OccupationIndustryYear, CountryDepartmentYear, MSAYear)
+                     OccupationIndustryYear, CountryCountryYear,
+                     CountryDepartmentYear, CountryMSAYear, MSAYear)
 from ..api_schemas import marshal
 from .routing import lookup_classification_level
 from .. import api_schemas as schemas
@@ -219,7 +220,17 @@ def eey_location_partners(entity_type, entity_id, buildingblock_level):
         q = CountryDepartmentYear.query\
             .filter_by(location_id=entity_id)\
             .all()
-        return marshal(schemas.country_department_year, q)
+        return marshal(schemas.country_x_year, q)
+    elif location_level == "msa":
+        q = CountryMSAYear.query\
+            .filter_by(location_id=entity_id)\
+            .all()
+        return marshal(schemas.country_x_year, q)
+    elif location_level == "country":
+        q = CountryCountryYear.query\
+            .filter_by(location_id=entity_id)\
+            .all()
+        return marshal(schemas.country_x_year, q)
     else:
         msg = "Data doesn't exist at location level {}"\
             .format(location_level)

colombia/datasets.py

@@ -320,6 +320,53 @@ def load_trade4digit_municipality():
 }
 
 
+def read_trade4digit_rcpy_country():
+    df = pd.read_stata(prefix_path("Trade/exp_rcpy_rc_p4.dta"))
+    df["r"] = "COL"
+    return df
+
+
+trade4digit_rcpy_country = {
+    "read_function": read_trade4digit_rcpy_country,
+    "field_mapping": {
+        "r": "location",
+        "ctry_dest": "country",
+        "p": "product",
+        "yr": "year",
+        "X_rcpy_d": "export_value",
+        "NP_rcpy": "export_num_plants"
+    },
+    "classification_fields": {
+        "location": {
+            "classification": location_classification,
+            "level": "country"
+        },
+        "product": {
+            "classification": product_classification,
+            "level": "4digit"
+        },
+        "country": {
+            "classification": country_classification,
+            "level": "country"
+        },
+    },
+    "digit_padding": {
+        "country": 3,
+        "product": 4
+    },
+    "facet_fields": ["location", "country", "product", "year"],
+    "facets": {
+        ("country_id", "location_id", "year"): {
+            "export_value": sum_group,
+            "export_num_plants": sum_group
+        },
+        ("country_id", "location_id", "product_id", "year"): {
+            "export_value": first,
+            "export_num_plants": first
+        }
+    }
+}
+
 trade4digit_rcpy_department = {
     "read_function": lambda: pd.read_stata(prefix_path("Trade/exp_rcpy_r2_p4.dta")),
     "field_mapping": {
@@ -352,8 +399,8 @@ def load_trade4digit_municipality():
     "facet_fields": ["location", "country", "product", "year"],
     "facets": {
         ("country_id", "location_id", "year"): {
-            "export_value": first,
-            "export_num_plants": first
+            "export_value": sum_group,
+            "export_num_plants": sum_group
         },
         ("country_id", "location_id", "product_id", "year"): {
             "export_value": first,
@@ -363,6 +410,43 @@ def load_trade4digit_municipality():
 }
 
 
+trade4digit_rcpy_msa = {
+    "read_function": lambda: pd.read_stata(prefix_path("Trade/exp_rcpy_ra_p4.dta")),
+    "field_mapping": {
+        "r": "location",
+        "ctry_dest": "country",
+        "p": "product",
+        "yr": "year",
+        "X_rcpy_d": "export_value",
+        "NP_rcpy": "export_num_plants"
+    },
+    "classification_fields": {
+        "location": {
+            "classification": location_classification,
+            "level": "msa"
+        },
+        "product": {
+            "classification": product_classification,
+            "level": "4digit"
+        },
+        "country": {
+            "classification": country_classification,
+            "level": "country"
+        },
+    },
+    "digit_padding": {
+        "country": 3,
+        "product": 4
+    },
+    "facet_fields": ["location", "country", "product", "year"],
+    "facets": {
+        ("country_id", "location_id", "year"): {
+            "export_value": sum_group,
+            "export_num_plants": sum_group
+        }
+    }
+}
+
 trade4digit_rcpy_municipality = {
     "read_function": lambda: pd.read_stata(prefix_path("Trade/exp_rcpy_r5_p4.dta")),
     "field_mapping": {

colombia/import.py

@@ -10,7 +10,8 @@
                       industry4digit_msa, industry2digit_department,
                       industry4digit_municipality,
                       trade4digit_rcpy_municipality,
-                      trade4digit_rcpy_department, population,
+                      trade4digit_rcpy_department, trade4digit_rcpy_msa,
+                      trade4digit_rcpy_country, population,
                       gdp_nominal_department, gdp_real_department,
                       occupation2digit, occupation2digit_industry2digit)
 
@@ -130,6 +131,21 @@
             df.to_sql("msa_year", db.engine, index=False,
                       chunksize=10000, if_exists="append")
 
+            # Country - trade rcpy
+            ret = process_dataset(trade4digit_rcpy_country)
+
+            df = ret[("country_id", "location_id", "year")].reset_index()
+            df.to_sql("country_country_year", db.engine,
+                      index=False, chunksize=10000, if_exists="append")
+
+
+            # MSA - trade rcpy
+            ret = process_dataset(trade4digit_rcpy_msa)
+
+            df = ret[("country_id", "location_id", "year")].reset_index()
+            df.to_sql("country_msa_year", db.engine,
+                      index=False, chunksize=10000, if_exists="append")
+
             # Municipality - trade rcpy
             ret = process_dataset(trade4digit_rcpy_municipality)
 

colombia/models.py

@@ -5,4 +5,5 @@
                           DepartmentIndustryYear, IndustryYear,
                           MunicipalityIndustryYear, MSAProductYear,
                           MSAIndustryYear, OccupationYear,
-                          OccupationIndustryYear, CountryDepartmentYear, MSAYear)
+                          OccupationIndustryYear, CountryCountryYear,
+                          CountryDepartmentYear, CountryMSAYear, MSAYear)