diff --git a/Gemfile b/Gemfile index 3c74edf0..318939ac 100644 --- a/Gemfile +++ b/Gemfile @@ -12,8 +12,6 @@ gem "sqlite3" gem "puma" gem "redis" -gem "airrecord" # Airtable client - # Assets gem "sprockets-rails" gem "sass-rails" diff --git a/Gemfile.lock b/Gemfile.lock index 32c2db10..424f6985 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -490,7 +490,6 @@ PLATFORMS DEPENDENCIES active_storage_validations - airrecord appsignal audits1984 aws-sdk-locationservice diff --git a/app/models/airtable/hackathon.rb b/app/models/airtable/hackathon.rb deleted file mode 100644 index d0c00bf2..00000000 --- a/app/models/airtable/hackathon.rb +++ /dev/null @@ -1,133 +0,0 @@ -module Airtable - class Hackathon < Airrecord::Table - self.base_key = Rails.application.credentials.dig(:airtable, :hackathons_base) - self.table_name = "applications" - - def name - self["name"].strip - end - - def applicant_email - email = self["applicant_email"]&.strip || "hackathons+airtable_migration@hackclub.com" - return email if email.match? URI::MailTo::EMAIL_REGEXP - - # Some records have multiple emails separated by commas - email.split(",").first&.strip - end - - def starts_at - self["start"] - end - - def ends_at - self["end"] - end - - def website - url = URI.parse(self["website"].strip) - url = URI.parse("http://#{url}") unless url.scheme - url.to_s - end - - def expected_attendees - attendees = self["expected_attendance"] || 1 - (attendees > 0) ? attendees : 1 - end - - def apac - !!self["apac"] - end - - def created_at - self["created_at"] - end - - def modality - return ::Hackathon.modalities[:online] if self["virtual"] - return ::Hackathon.modalities[:hybrid] if self["hybrid"] - ::Hackathon.modalities[:in_person] - end - - def status - return ::Hackathon.statuses[:approved] if self["approved"] - return ::Hackathon.statuses[:rejected] if self["rejected"] - ::Hackathon.statuses[:pending] - end - - def high_school_led - !!self["Are you a high schooler?"] - end - - def offers_financial_assistance - !!self["Would you like to apply for financial assistance?"] - end - - def full_address - city = self["parsed_city"] - state = self["parsed_state_code"] - country = self["parsed_country"] - country_code = self["parsed_country_code"] - - [city, state, country || country_code].compact.join(", ") - end - - def swag_mailing_address - return nil unless offers_financial_assistance - - line1 = self["Address Line 1"] - line2 = self["Address Line 2"] - city = self["Mailing address (City)"] - province = self["Mailing Address (State)"] - postal_code = self["Mailing Address (Zip code)"] - country_code = ISO3166::Country.find_country_by_any_name(self["Mailing Address (Country)"])&.alpha2 - - invalid = [line1, city, country_code].any?(&:blank?) - if invalid - address = [] - address << [line1, line2].compact.join(" ") - address << city << province << postal_code << country_code - address = address.compact.join(", ") - - location = Geocoder.search(address).first - return nil unless location - - line1 = [location.house_number, location.street].compact.join(" ") - line2 = nil - city = location.city - province = location.province || location.state - postal_code = location.postal_code - country_code = location.country_code.upcase - end - - {line1:, line2:, city:, province:, postal_code:, country_code:} - end - - def coordinates - coords = [self["lat"], self["lng"]] - return nil if coords.any?(&:blank?) - - coords - end - - def logo - url = self["logo"]&.last&.[]("url") - return nil unless url.present? - URI.parse(url).open - end - - def logo_filename - self["logo"]&.last&.[]("filename") - end - - def banner - url = self["banner"]&.last&.[]("url") - return nil unless url.present? - - URI.parse(url).open - end - - def banner_filename - self["banner"]&.last&.[]("filename") - end - end -end diff --git a/app/models/airtable/subscriber.rb b/app/models/airtable/subscriber.rb deleted file mode 100644 index 7826ca74..00000000 --- a/app/models/airtable/subscriber.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Airtable - class Subscriber < Airrecord::Table - self.base_key = Rails.application.credentials.dig(:airtable, :hackathons_base) - self.table_name = "subscribers" - - def email - self["email"].strip - end - - def location - self["location"] - end - - def coordinates - coords = [self["latitude"], self["longitude"]] - return nil if coords.any?(&:blank?) - - coords - end - - def created_at - self["created_at"] - end - - def status - state = (self["unsubscribed"] == 0) ? :active : :inactive - ::Hackathon::Subscription.statuses[state] - end - - def unsubscribed_at - self["unsubscribed_at"] - end - end -end diff --git a/config/initializers/airrecord.rb b/config/initializers/airrecord.rb deleted file mode 100644 index 9adc7759..00000000 --- a/config/initializers/airrecord.rb +++ /dev/null @@ -1 +0,0 @@ -Airrecord.api_key = Rails.application.credentials.dig(:airtable, :api_key) diff --git a/doc/airtable_data_migration.md b/doc/airtable_data_migration.md deleted file mode 100644 index 6a15b959..00000000 --- a/doc/airtable_data_migration.md +++ /dev/null @@ -1,43 +0,0 @@ -# Airtable Data Migration - -Here are my notes on the data migration. - -- Imported records will have an `airtable_id` column -- `apac` field - - `false`: not APAC (imported from Airtable) - - `true`: APAC (imported from Airtable) - - `nil`: this record was NOT imported from Airtable, and Country gem will - decide whether it's APAC or not -- If a hackathon on Airtable has an empty email field, the default applicant - is `hackathons+airtable_migration@hackclub.com` -- If a hackathon on Airtable was missing expected attendees, it defaulted to 1 -- Modality (online, in-person, hybrid) were booleans on Airtable, meaning that - it could be possible to have more than one modality. To determine the true - modality, I used the following order of precedence: online (virtual), hybrid, - in-person. - This is the same order being used in the front-end. -- Same thing applies for status (approved, rejected, pending). Order: approved, - rejected, pending -- There are quite a bit of bad data I discovered and fix on Airtable before - running the migration - - End dates before start dates - - Invalid website URLs - - Invalid email addresses -- If there is invalid data for Subscriptions (a location that can't be - geocoded), then that Airtable record is skipped. -- The migration script is idempotent, meaning that you may run it multiple times - and it will not create duplicates. It also won't update existing records. - -Run it with - -```shell -rake airtable_data:migrate - -# or the following to skip hackathons (1st arg) or subscriptions (2nd arg) -rake "airtable_data:migrate[true, false]" -``` - -I'm estimate it'll take a little over an hour to run due to the rate limit on geocoding. -We have a total of 4,200 records that will need to be geocoded. - -~ @garyhtou diff --git a/lib/tasks/airtable_data.rake b/lib/tasks/airtable_data.rake deleted file mode 100644 index 16607bb0..00000000 --- a/lib/tasks/airtable_data.rake +++ /dev/null @@ -1,137 +0,0 @@ -namespace :airtable_data do - desc "Migrates data from Airtable to the database" - task :migrate, [:skip_hackathons, :skip_subscriptions] => [:environment] do |t, args| - def migrate_hackathons - airtable_swag_mailing_address_fields = [ - "Address Line 1", - "Address Line 2", - "Mailing address (City)", - "Mailing Address (State)", - "Mailing Address (Zip code)", - "Mailing Address (Country)" - ] - - puts "MIGRATING HACKATHONS" - Airtable::Hackathon.all(sort: {created_at: :asc}).each do |record| - if Hackathon.find_by(airtable_id: record.id) - puts "Skipping #{record.name} (#{record.id})" - next - end - - ActiveRecord::Base.transaction do - hackathon = Hackathon.new - hackathon.airtable_id = record.id - - %w[name status starts_at ends_at website high_school_led - expected_attendees modality apac].each do |field| - value = record.send field - hackathon.send :"#{field}=", value - end - - # Re-geocoding existing address - hackathon.address = record.full_address - - # Attach logo and banner images - hackathon.logo.attach(io: record.logo, filename: record.logo_filename) if record.logo - hackathon.banner.attach(io: record.banner, filename: record.banner_filename) if record.banner - - hackathon.send(:record, :imported_from_airtable, data: record.fields.except(airtable_swag_mailing_address_fields)) - - # Financial assistance - if record.offers_financial_assistance - hackathon.tag_with("Offers Financial Assistance") - end - - # Swag mailing address - if record.swag_mailing_address - hackathon.swag_mailing_address_attributes = { - **record.swag_mailing_address, - created_at: record.created_at - } - - original_address = record.fields.slice(airtable_swag_mailing_address_fields) - - hackathon.swag_mailing_address.send(:record, :imported_from_airtable, data: original_address) - end - - # Applicant - hackathon.applicant = User.find_or_initialize_by(email_address: record.applicant_email) do |applicant| - applicant.created_at = record.created_at - applicant.send(:record, :imported_from_airtable) - end - - # Save it! - puts "Creating #{hackathon.name} (#{hackathon.airtable_id})" - hackathon.created_at = record.created_at - hackathon.save! - - # Correct :created Event's created_at - [hackathon, hackathon.swag_mailing_address].compact.each do |object| - object.events.find_by(action: :created).update!(created_at: hackathon.created_at) - end - - # Flag geocoding errors - if hackathon.in_person? && !hackathon.geocoded? - hackathon.send(:record, :airtable_migration_geocoding_error, data: {airtable_coordinates: record.coordinates}) - end - if record.coordinates && hackathon.geocoded? && hackathon.distance_to(record.coordinates) > 100 - hackathon.send(:record, :airtable_migration_geocoding_distance_error, data: {airtable_coordinates: record.coordinates}) - end - end - end - end - - def migrate_subscriptions - puts "MIGRATING SUBSCRIPTIONS" - Airtable::Subscriber.all(sort: {created_at: :asc}).each do |record| - if (subscription = Hackathon::Subscription.find_by(airtable_id: record.id)) - puts "Skipping #{record.id} (#{subscription.location})" - next - end - - ActiveRecord::Base.transaction do - subscription = Hackathon::Subscription.new - subscription.airtable_id = record.id - - # Re-geocoding existing address - subscription.location_input = record.location - - subscription.status = record.status - if subscription.inactive? && record.unsubscribed_at - subscription.send(:record, :disabled, created_at: record.unsubscribed_at) - end - - subscription.send(:record, :imported_from_airtable, data: record.fields) - - # Subscriber - subscription.subscriber = User.find_or_initialize_by(email_address: record.email) do |subscriber| - subscriber.created_at = record.created_at - subscriber.send(:record, :imported_from_airtable) - end - - # Save it! - subscription.created_at = record.created_at - subscription.save! - puts "Creating #{subscription.airtable_id} (#{subscription.location})" - - # Correct :created Event's created_at - subscription.events.find_by(action: :created).update!(created_at: subscription.created_at) - end - rescue ActiveRecord::RecordInvalid => e - puts "Skipping #{record.id} (#{record.location}) because of validation error: #{e.message}" - end - end - - if args[:skip_hackathons] == "true" - puts "Skipping hackathons migration" - else - migrate_hackathons - end - - if args[:skip_subscriptions] == "true" - puts "Skipping subscriptions migration" - else - migrate_subscriptions - end - end -end