From 84278c761b127acedf24bbcaf36089959ed1ddbd Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Fri, 7 Feb 2025 09:23:37 +0000 Subject: [PATCH 01/12] Add exchange and currency fields to trade imports --- .../import/configurations_controller.rb | 1 + app/controllers/import/rows_controller.rb | 2 +- app/helpers/imports_helper.rb | 1 + app/models/import.rb | 1 + app/models/import/row.rb | 1 + app/models/trade_import.rb | 26 +++++++++++++------ .../configurations/_trade_import.html.erb | 2 ++ ...xchange_and_currency_columns_to_imports.rb | 5 ++++ ...add_exchange_and_currency_to_securities.rb | 6 +++++ ...50207062248_add_exchange_to_import_rows.rb | 5 ++++ db/schema.rb | 10 +++++-- 11 files changed, 49 insertions(+), 11 deletions(-) create mode 100644 db/migrate/20250207061453_add_exchange_and_currency_columns_to_imports.rb create mode 100644 db/migrate/20250207061552_add_exchange_and_currency_to_securities.rb create mode 100644 db/migrate/20250207062248_add_exchange_to_import_rows.rb diff --git a/app/controllers/import/configurations_controller.rb b/app/controllers/import/configurations_controller.rb index b1ae9e50..4ffec2c0 100644 --- a/app/controllers/import/configurations_controller.rb +++ b/app/controllers/import/configurations_controller.rb @@ -29,6 +29,7 @@ class Import::ConfigurationsController < ApplicationController :account_col_label, :qty_col_label, :ticker_col_label, + :exchange_col_label, :price_col_label, :entity_type_col_label, :notes_col_label, diff --git a/app/controllers/import/rows_controller.rb b/app/controllers/import/rows_controller.rb index b5b9092c..8be66680 100644 --- a/app/controllers/import/rows_controller.rb +++ b/app/controllers/import/rows_controller.rb @@ -14,7 +14,7 @@ class Import::RowsController < ApplicationController private def row_params - params.require(:import_row).permit(:type, :account, :date, :qty, :ticker, :price, :amount, :currency, :name, :category, :tags, :entity_type, :notes) + params.require(:import_row).permit(:type, :account, :date, :qty, :ticker, :exchange, :price, :amount, :currency, :name, :category, :tags, :entity_type, :notes) end def set_import_row diff --git a/app/helpers/imports_helper.rb b/app/helpers/imports_helper.rb index a202a697..7f8a65f0 100644 --- a/app/helpers/imports_helper.rb +++ b/app/helpers/imports_helper.rb @@ -20,6 +20,7 @@ module ImportsHelper notes: "Notes", qty: "Quantity", ticker: "Ticker", + exchange: "Exchange", price: "Price", entity_type: "Type" }[key] diff --git a/app/models/import.rb b/app/models/import.rb index 15646d44..90ca4c42 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -73,6 +73,7 @@ class Import < ApplicationRecord date: row[date_col_label].to_s, qty: sanitize_number(row[qty_col_label]).to_s, ticker: row[ticker_col_label].to_s, + exchange: row[exchange_col_label]&.strip.to_s, price: sanitize_number(row[price_col_label]).to_s, amount: sanitize_number(row[amount_col_label]).to_s, currency: (row[currency_col_label] || default_currency).to_s, diff --git a/app/models/import/row.rb b/app/models/import/row.rb index d4316a60..09c5d692 100644 --- a/app/models/import/row.rb +++ b/app/models/import/row.rb @@ -3,6 +3,7 @@ class Import::Row < ApplicationRecord validates :amount, numericality: true, allow_blank: true validates :currency, presence: true + validates :exchange, presence: true, if: -> { import.type == "TradeImport" && import.exchange_col_label.present? } validate :date_valid validate :required_columns diff --git a/app/models/trade_import.rb b/app/models/trade_import.rb index 4ca57ea1..e4a9ca08 100644 --- a/app/models/trade_import.rb +++ b/app/models/trade_import.rb @@ -5,14 +5,24 @@ class TradeImport < Import rows.each do |row| account = mappings.accounts.mappable_for(row.account) - security = Security.find_or_create_by(ticker: row.ticker) + security = Security.find_or_create_by!( + ticker: row.ticker, + exchange_mic: row.exchange.presence || "UNKNOWN", + exchange_acronym: row.exchange.presence || "Unknown", + currency: row.currency.presence || account.currency + ) entry = account.entries.build \ date: row.date_iso, amount: row.signed_amount, name: row.name, - currency: row.currency, - entryable: Account::Trade.new(security: security, qty: row.qty, currency: row.currency, price: row.price), + currency: row.currency.presence || account.currency, + entryable: Account::Trade.new( + security: security, + qty: row.qty, + currency: row.currency.presence || account.currency, + price: row.price + ), import: self entry.save! @@ -29,7 +39,7 @@ class TradeImport < Import end def column_keys - %i[date ticker qty price currency account name] + %i[date ticker exchange currency qty price account name] end def dry_run @@ -41,10 +51,10 @@ class TradeImport < Import def csv_template template = <<-CSV - date*,ticker*,qty*,price*,currency,account,name - 05/15/2024,AAPL,10,150.00,USD,Trading Account,Apple Inc. Purchase - 05/16/2024,GOOGL,-5,2500.00,USD,Investment Account,Alphabet Inc. Sale - 05/17/2024,TSLA,2,700.50,USD,Retirement Account,Tesla Inc. Purchase + date*,ticker*,exchange,currency,qty*,price*,account,name + 05/15/2024,AAPL,XNAS,USD,10,150.00,Trading Account,Apple Inc. Purchase + 05/16/2024,GOOGL,XNAS,USD,-5,2500.00,Investment Account,Alphabet Inc. Sale + 05/17/2024,TSLA,XNAS,USD,2,700.50,Retirement Account,Tesla Inc. Purchase CSV CSV.parse(template, headers: true) diff --git a/app/views/import/configurations/_trade_import.html.erb b/app/views/import/configurations/_trade_import.html.erb index 278debf8..231688d8 100644 --- a/app/views/import/configurations/_trade_import.html.erb +++ b/app/views/import/configurations/_trade_import.html.erb @@ -12,6 +12,8 @@ <%= form.select :ticker_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Ticker" } %> + <%= form.select :exchange_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Exchange" } %> + <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %> <%= form.select :price_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Price" } %> <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" } %> <%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" } %> diff --git a/db/migrate/20250207061453_add_exchange_and_currency_columns_to_imports.rb b/db/migrate/20250207061453_add_exchange_and_currency_columns_to_imports.rb new file mode 100644 index 00000000..76c25119 --- /dev/null +++ b/db/migrate/20250207061453_add_exchange_and_currency_columns_to_imports.rb @@ -0,0 +1,5 @@ +class AddExchangeAndCurrencyColumnsToImports < ActiveRecord::Migration[7.2] + def change + add_column :imports, :exchange_col_label, :string + end +end diff --git a/db/migrate/20250207061552_add_exchange_and_currency_to_securities.rb b/db/migrate/20250207061552_add_exchange_and_currency_to_securities.rb new file mode 100644 index 00000000..be176e69 --- /dev/null +++ b/db/migrate/20250207061552_add_exchange_and_currency_to_securities.rb @@ -0,0 +1,6 @@ +class AddExchangeAndCurrencyToSecurities < ActiveRecord::Migration[7.2] + def change + add_column :securities, :currency, :string + add_index :securities, :currency + end +end diff --git a/db/migrate/20250207062248_add_exchange_to_import_rows.rb b/db/migrate/20250207062248_add_exchange_to_import_rows.rb new file mode 100644 index 00000000..ec305185 --- /dev/null +++ b/db/migrate/20250207062248_add_exchange_to_import_rows.rb @@ -0,0 +1,5 @@ +class AddExchangeToImportRows < ActiveRecord::Migration[7.2] + def change + add_column :import_rows, :exchange, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index 056caf8c..b08bf4db 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2025_02_06_151825) do +ActiveRecord::Schema[7.2].define(version: 2025_02_07_062248) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -102,7 +102,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_06_151825) do t.decimal "balance", precision: 19, scale: 4 t.string "currency" t.boolean "is_active", default: true, null: false - t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true + t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true t.uuid "import_id" t.uuid "plaid_account_id" t.boolean "scheduled_for_deletion", default: false @@ -386,6 +386,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_06_151825) do t.text "notes" t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.string "exchange" t.index ["import_id"], name: "index_import_rows_on_import_id" end @@ -415,6 +416,9 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_06_151825) do t.string "date_format", default: "%m/%d/%Y" t.string "signage_convention", default: "inflows_positive" t.string "error" + t.string "currency", default: "USD" + t.string "number_format", default: "1,234.56" + t.string "exchange_col_label" t.index ["family_id"], name: "index_imports_on_family_id" end @@ -550,7 +554,9 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_06_151825) do t.string "exchange_mic" t.string "exchange_acronym" t.string "logo_url" + t.string "currency" t.index ["country_code"], name: "index_securities_on_country_code" + t.index ["currency"], name: "index_securities_on_currency" t.index ["ticker", "exchange_mic"], name: "index_securities_on_ticker_and_exchange_mic", unique: true end -- 2.53.0 From 015e614be134430a973ea7790628666424b7a73a Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Sat, 15 Feb 2025 16:10:31 +0000 Subject: [PATCH 02/12] Add exchange_operating_mic support for trade imports - Added required columns and updated models --- app/helpers/imports_helper.rb | 17 +++++++ app/models/import.rb | 26 +++++----- app/models/import/row.rb | 18 +++++++ app/models/trade_import.rb | 47 +++++++++++++++++-- .../configurations/_trade_import.html.erb | 46 +++++++++++------- ...ange_operating_mic_col_label_to_imports.rb | 16 +++++++ ...d_exchange_operating_mic_to_import_rows.rb | 5 ++ db/schema.rb | 7 +-- 8 files changed, 143 insertions(+), 39 deletions(-) create mode 100644 db/migrate/20250215153930_add_exchange_operating_mic_col_label_to_imports.rb create mode 100644 db/migrate/20250215160141_add_exchange_operating_mic_to_import_rows.rb diff --git a/app/helpers/imports_helper.rb b/app/helpers/imports_helper.rb index 67930cc1..87a58460 100644 --- a/app/helpers/imports_helper.rb +++ b/app/helpers/imports_helper.rb @@ -55,6 +55,23 @@ module ImportsHelper [ base, border ].join(" ") end + def import_col_labels + { + date: "Date", + ticker: "Ticker", + exchange_operating_mic: "Exchange Operating MIC", + currency: "Currency", + qty: "Quantity", + price: "Price", + account: "Account", + name: "Name", + category: "Category", + tags: "Tags", + entity_type: "Entity Type", + notes: "Notes" + } + end + private def permitted_import_types %w[transaction_import trade_import account_import mint_import] diff --git a/app/models/import.rb b/app/models/import.rb index 56b90243..8d3da334 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -107,19 +107,19 @@ class Import < ApplicationRecord csv_rows.each do |row| rows.create!( - account: row[account_col_label].to_s, - date: row[date_col_label].to_s, - qty: sanitize_number(row[qty_col_label]).to_s, - ticker: row[ticker_col_label].to_s, - exchange: row[exchange_col_label]&.strip.to_s, - price: sanitize_number(row[price_col_label]).to_s, - amount: sanitize_number(row[amount_col_label]).to_s, - currency: (row[currency_col_label] || default_currency).to_s, - name: (row[name_col_label] || default_row_name).to_s, - category: row[category_col_label].to_s, - tags: row[tags_col_label].to_s, - entity_type: row[entity_type_col_label].to_s, - notes: row[notes_col_label].to_s + account: row[account_col_label]&.strip.to_s, + date: row[date_col_label]&.strip.to_s, + qty: sanitize_number(row[qty_col_label])&.strip.to_s, + ticker: row[ticker_col_label]&.strip.to_s, + exchange_operating_mic: row[exchange_operating_mic_col_label]&.strip.to_s, + price: sanitize_number(row[price_col_label])&.strip.to_s, + amount: sanitize_number(row[amount_col_label])&.strip.to_s, + currency: (row[currency_col_label] || default_currency)&.strip.to_s, + name: (row[name_col_label] || default_row_name)&.strip.to_s, + category: row[category_col_label]&.strip.to_s, + tags: row[tags_col_label]&.strip.to_s, + entity_type: row[entity_type_col_label]&.strip.to_s, + notes: row[notes_col_label]&.strip.to_s ) end end diff --git a/app/models/import/row.rb b/app/models/import/row.rb index 09c5d692..8c50890a 100644 --- a/app/models/import/row.rb +++ b/app/models/import/row.rb @@ -4,10 +4,12 @@ class Import::Row < ApplicationRecord validates :amount, numericality: true, allow_blank: true validates :currency, presence: true validates :exchange, presence: true, if: -> { import.type == "TradeImport" && import.exchange_col_label.present? } + validates :exchange_operating_mic, presence: true, if: -> { import.type == "TradeImport" && import.exchange_operating_mic_col_label.present? } validate :date_valid validate :required_columns validate :currency_is_valid + validate :exchange_operating_mic_is_valid, if: -> { import.type == "TradeImport" && exchange_operating_mic.present? } scope :ordered, -> { order(:id) } @@ -82,4 +84,20 @@ class Import::Row < ApplicationRecord errors.add(:currency, "is not a valid currency code") end end + + def exchange_operating_mic_is_valid + return true if exchange_operating_mic.blank? + return true unless Security.security_prices_provider.present? + + response = Security.security_prices_provider.fetch_security_prices( + ticker: ticker, + mic_code: exchange_operating_mic, + start_date: Date.current, + end_date: Date.current + ) + + unless response.success? + errors.add(:exchange_operating_mic, "is not valid for ticker #{ticker}. No prices found for this combination.") + end + end end diff --git a/app/models/trade_import.rb b/app/models/trade_import.rb index e4a9ca08..75dca8c4 100644 --- a/app/models/trade_import.rb +++ b/app/models/trade_import.rb @@ -5,10 +5,11 @@ class TradeImport < Import rows.each do |row| account = mappings.accounts.mappable_for(row.account) - security = Security.find_or_create_by!( + + # Try to find or create security with exchange validation + security = find_or_create_security( ticker: row.ticker, - exchange_mic: row.exchange.presence || "UNKNOWN", - exchange_acronym: row.exchange.presence || "Unknown", + exchange_operating_mic: row.exchange_operating_mic, currency: row.currency.presence || account.currency ) @@ -39,7 +40,7 @@ class TradeImport < Import end def column_keys - %i[date ticker exchange currency qty price account name] + %i[date ticker exchange_operating_mic currency qty price account name] end def dry_run @@ -51,7 +52,7 @@ class TradeImport < Import def csv_template template = <<-CSV - date*,ticker*,exchange,currency,qty*,price*,account,name + date*,ticker*,exchange_operating_mic,currency,qty*,price*,account,name 05/15/2024,AAPL,XNAS,USD,10,150.00,Trading Account,Apple Inc. Purchase 05/16/2024,GOOGL,XNAS,USD,-5,2500.00,Investment Account,Alphabet Inc. Sale 05/17/2024,TSLA,XNAS,USD,2,700.50,Retirement Account,Tesla Inc. Purchase @@ -59,4 +60,40 @@ class TradeImport < Import CSV.parse(template, headers: true) end + + private + + def find_or_create_security(ticker:, exchange_operating_mic:, currency:) + # First try to find an existing security + security = Security.find_by( + ticker: ticker, + exchange_operating_mic: exchange_operating_mic + ) + + return security if security.present? + + # Create new security + security = Security.new( + ticker: ticker, + exchange_operating_mic: exchange_operating_mic + ) + + # Only validate with Synth if exchange_operating_mic is provided and Synth is configured + if exchange_operating_mic.present? && Security.security_prices_provider.present? + response = Security.security_prices_provider.fetch_security_prices( + ticker: ticker, + mic_code: exchange_operating_mic, + start_date: Date.current, + end_date: Date.current + ) + + if !response.success? + raise ImportError, "Unable to validate security #{ticker} on exchange #{exchange_operating_mic}. Prices could not be found." + end + end + + # If we get here, either no exchange was provided or validation passed + security.save! + security + end end diff --git a/app/views/import/configurations/_trade_import.html.erb b/app/views/import/configurations/_trade_import.html.erb index 500f0dc9..a8d056b0 100644 --- a/app/views/import/configurations/_trade_import.html.erb +++ b/app/views/import/configurations/_trade_import.html.erb @@ -1,27 +1,37 @@ <%# locals: (import:) %> <%= styled_form_with model: @import, url: import_configuration_path(@import), scope: :import, method: :patch, class: "space-y-2" do |form| %> -
- <%= form.select :date_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Date" }, required: true %> - <%= form.select :date_format, Family::DATE_FORMATS, { label: t(".date_format_label")}, label: true, required: true %> -
+
+
+ <%= form.select :date_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Date" }, required: true %> + <%= form.select :date_format, Family::DATE_FORMATS, { label: t(".date_format_label")}, label: true, required: true %> +
-
- <%= form.select :qty_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Quantity" } %> - <%= form.select :signage_convention, [["Buys are positive qty", "inflows_positive"], ["Buys are negative qty", "inflows_negative"]], label: true %> -
+
+ <%= form.select :qty_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Quantity" } %> + <%= form.select :signage_convention, [["Buys are positive qty", "inflows_positive"], ["Buys are negative qty", "inflows_negative"]], label: true %> +
-
- <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %> - <%= form.select :number_format, Import::NUMBER_FORMATS.keys, { label: "Format", prompt: "Select format" }, required: true %> -
+
+ <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %> + <%= form.select :number_format, Import::NUMBER_FORMATS.keys, { label: "Format", prompt: "Select format" }, required: true %> +
- <%= form.select :ticker_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Ticker" } %> - <%= form.select :exchange_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Exchange" } %> - <%= form.select :currency_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Currency" } %> - <%= form.select :price_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Price" } %> - <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" } %> - <%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" } %> + <%= form.select :ticker_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Ticker" } %> + <%= form.select :exchange_operating_mic_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Exchange Operating MIC" } %> + <%= form.select :price_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Price" } %> + <%= form.select :account_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Account (optional)" } %> + <%= form.select :name_col_label, import.csv_headers, { include_blank: "Leave empty", label: "Name (optional)" } %> + + <% if Security.security_prices_provider.nil? %> +
+

+ Note: The Synth provider is not configured. Exchange validation is disabled. + Securities will be created without exchange validation, and price history will not be available. +

+
+ <% end %> +
<%= form.submit "Apply configuration", class: "w-full btn btn--primary", disabled: import.complete? %> <% end %> diff --git a/db/migrate/20250215153930_add_exchange_operating_mic_col_label_to_imports.rb b/db/migrate/20250215153930_add_exchange_operating_mic_col_label_to_imports.rb new file mode 100644 index 00000000..7e9d6d74 --- /dev/null +++ b/db/migrate/20250215153930_add_exchange_operating_mic_col_label_to_imports.rb @@ -0,0 +1,16 @@ +class AddExchangeOperatingMicColLabelToImports < ActiveRecord::Migration[7.2] + def up + add_column :imports, :exchange_operating_mic_col_label, :string + + # Migrate existing trade imports to use the new column + Import.where(type: "TradeImport").find_each do |import| + if import.exchange_col_label.present? + import.update_column(:exchange_operating_mic_col_label, import.exchange_col_label) + end + end + end + + def down + remove_column :imports, :exchange_operating_mic_col_label + end +end diff --git a/db/migrate/20250215160141_add_exchange_operating_mic_to_import_rows.rb b/db/migrate/20250215160141_add_exchange_operating_mic_to_import_rows.rb new file mode 100644 index 00000000..34951bf1 --- /dev/null +++ b/db/migrate/20250215160141_add_exchange_operating_mic_to_import_rows.rb @@ -0,0 +1,5 @@ +class AddExchangeOperatingMicToImportRows < ActiveRecord::Migration[7.2] + def change + add_column :import_rows, :exchange_operating_mic, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index ad7593a6..0e92310b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,8 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. - -ActiveRecord::Schema[7.2].define(version: 2025_02_12_163624) do +ActiveRecord::Schema[7.2].define(version: 2025_02_15_160141) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -387,6 +386,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_12_163624) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.string "exchange" + t.string "exchange_operating_mic" t.index ["import_id"], name: "index_import_rows_on_import_id" end @@ -419,6 +419,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_12_163624) do t.string "currency", default: "USD" t.string "number_format", default: "1,234.56" t.string "exchange_col_label" + t.string "exchange_operating_mic_col_label" t.index ["family_id"], name: "index_imports_on_family_id" end @@ -546,7 +547,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_12_163624) do t.index ["outflow_transaction_id"], name: "index_rejected_transfers_on_outflow_transaction_id" end -create_table "securities", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| + create_table "securities", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t| t.string "ticker", null: false t.string "name" t.datetime "created_at", null: false -- 2.53.0 From 016a6f910e7dc045022b3a29019a46e2c63cda19 Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Wed, 19 Feb 2025 09:22:20 +0000 Subject: [PATCH 03/12] refactor: remove exchange and currency columns --- .../import/configurations_controller.rb | 2 +- app/controllers/import/rows_controller.rb | 2 +- app/models/import/row.rb | 18 ----- app/models/provider/synth.rb | 5 +- app/models/security.rb | 3 +- app/models/trade_import.rb | 67 ++++++++++--------- ...ange_operating_mic_col_label_to_imports.rb | 7 -- ...084803_remove_exchange_from_import_rows.rb | 5 ++ ..._remove_exchange_col_label_from_imports.rb | 5 ++ ...9085043_remove_currency_from_securities.rb | 6 ++ db/schema.rb | 6 +- 11 files changed, 60 insertions(+), 66 deletions(-) create mode 100644 db/migrate/20250219084803_remove_exchange_from_import_rows.rb create mode 100644 db/migrate/20250219084839_remove_exchange_col_label_from_imports.rb create mode 100644 db/migrate/20250219085043_remove_currency_from_securities.rb diff --git a/app/controllers/import/configurations_controller.rb b/app/controllers/import/configurations_controller.rb index 4ffec2c0..9a060f8f 100644 --- a/app/controllers/import/configurations_controller.rb +++ b/app/controllers/import/configurations_controller.rb @@ -29,7 +29,7 @@ class Import::ConfigurationsController < ApplicationController :account_col_label, :qty_col_label, :ticker_col_label, - :exchange_col_label, + :exchange_operating_mic_col_label, :price_col_label, :entity_type_col_label, :notes_col_label, diff --git a/app/controllers/import/rows_controller.rb b/app/controllers/import/rows_controller.rb index 8be66680..b5b9092c 100644 --- a/app/controllers/import/rows_controller.rb +++ b/app/controllers/import/rows_controller.rb @@ -14,7 +14,7 @@ class Import::RowsController < ApplicationController private def row_params - params.require(:import_row).permit(:type, :account, :date, :qty, :ticker, :exchange, :price, :amount, :currency, :name, :category, :tags, :entity_type, :notes) + params.require(:import_row).permit(:type, :account, :date, :qty, :ticker, :price, :amount, :currency, :name, :category, :tags, :entity_type, :notes) end def set_import_row diff --git a/app/models/import/row.rb b/app/models/import/row.rb index 8c50890a..b46fe356 100644 --- a/app/models/import/row.rb +++ b/app/models/import/row.rb @@ -3,13 +3,11 @@ class Import::Row < ApplicationRecord validates :amount, numericality: true, allow_blank: true validates :currency, presence: true - validates :exchange, presence: true, if: -> { import.type == "TradeImport" && import.exchange_col_label.present? } validates :exchange_operating_mic, presence: true, if: -> { import.type == "TradeImport" && import.exchange_operating_mic_col_label.present? } validate :date_valid validate :required_columns validate :currency_is_valid - validate :exchange_operating_mic_is_valid, if: -> { import.type == "TradeImport" && exchange_operating_mic.present? } scope :ordered, -> { order(:id) } @@ -84,20 +82,4 @@ class Import::Row < ApplicationRecord errors.add(:currency, "is not a valid currency code") end end - - def exchange_operating_mic_is_valid - return true if exchange_operating_mic.blank? - return true unless Security.security_prices_provider.present? - - response = Security.security_prices_provider.fetch_security_prices( - ticker: ticker, - mic_code: exchange_operating_mic, - start_date: Date.current, - end_date: Date.current - ) - - unless response.success? - errors.add(:exchange_operating_mic, "is not valid for ticker #{ticker}. No prices found for this combination.") - end - end end diff --git a/app/models/provider/synth.rb b/app/models/provider/synth.rb index 65deaae5..829a5b7f 100644 --- a/app/models/provider/synth.rb +++ b/app/models/provider/synth.rb @@ -128,11 +128,12 @@ class Provider::Synth raw_response: error end - def search_securities(query:, dataset: "limited", country_code:) + def search_securities(query:, dataset: "limited", country_code: nil, exchange_operating_mic: nil) response = client.get("#{base_url}/tickers/search") do |req| req.params["name"] = query req.params["dataset"] = dataset - req.params["country_code"] = country_code + req.params["country_code"] = country_code if country_code.present? + req.params["exchange_operating_mic"] = exchange_operating_mic if exchange_operating_mic.present? req.params["limit"] = 25 end diff --git a/app/models/security.rb b/app/models/security.rb index 4a2bd1a7..46672fb5 100644 --- a/app/models/security.rb +++ b/app/models/security.rb @@ -13,7 +13,8 @@ class Security < ApplicationRecord security_prices_provider.search_securities( query: query[:search], dataset: "limited", - country_code: query[:country] + country_code: query[:country], + exchange_operating_mic: query[:exchange_operating_mic] ).securities.map { |attrs| new(**attrs) } end end diff --git a/app/models/trade_import.rb b/app/models/trade_import.rb index 75dca8c4..92e89cb7 100644 --- a/app/models/trade_import.rb +++ b/app/models/trade_import.rb @@ -9,8 +9,7 @@ class TradeImport < Import # Try to find or create security with exchange validation security = find_or_create_security( ticker: row.ticker, - exchange_operating_mic: row.exchange_operating_mic, - currency: row.currency.presence || account.currency + exchange_operating_mic: row.exchange_operating_mic ) entry = account.entries.build \ @@ -62,38 +61,44 @@ class TradeImport < Import end private + def find_or_create_security(ticker:, exchange_operating_mic:) + # Cache provider responses so that when we're looping through rows and importing, + # we only hit our provider for the unique combinations of ticker / exchange_operating_mic + cache_key = [ ticker, exchange_operating_mic ] + @provider_securities_cache ||= {} - def find_or_create_security(ticker:, exchange_operating_mic:, currency:) - # First try to find an existing security - security = Security.find_by( - ticker: ticker, - exchange_operating_mic: exchange_operating_mic - ) - - return security if security.present? - - # Create new security - security = Security.new( - ticker: ticker, - exchange_operating_mic: exchange_operating_mic - ) - - # Only validate with Synth if exchange_operating_mic is provided and Synth is configured - if exchange_operating_mic.present? && Security.security_prices_provider.present? - response = Security.security_prices_provider.fetch_security_prices( - ticker: ticker, - mic_code: exchange_operating_mic, - start_date: Date.current, - end_date: Date.current - ) - - if !response.success? - raise ImportError, "Unable to validate security #{ticker} on exchange #{exchange_operating_mic}. Prices could not be found." + provider_security = @provider_securities_cache[cache_key] ||= begin + if Security.security_prices_provider.present? + begin + response = Security.security_prices_provider.search_securities( + query: ticker, + exchange_operating_mic: exchange_operating_mic + ) + response.success? ? response.securities.first : nil + rescue StandardError => e + Rails.logger.error "Failed to fetch security data: #{e.message}" + nil + end end end - # If we get here, either no exchange was provided or validation passed - security.save! - security + if provider_security&.[](:ticker) && provider_security&.[](:exchange_operating_mic) + Security.find_or_create_by!( + ticker: provider_security[:ticker], + exchange_operating_mic: provider_security[:exchange_operating_mic] + ) do |security| + security.name = provider_security[:name] + security.country_code = provider_security[:country_code] + security.logo_url = provider_security[:logo_url] + security.exchange_acronym = provider_security[:exchange_acronym] + security.exchange_mic = provider_security[:exchange_mic] + end + else + # If provider data is not available, create security with just the ticker and exchange + Security.find_or_create_by!( + ticker: ticker, + exchange_operating_mic: exchange_operating_mic + ) + end end end diff --git a/db/migrate/20250215153930_add_exchange_operating_mic_col_label_to_imports.rb b/db/migrate/20250215153930_add_exchange_operating_mic_col_label_to_imports.rb index 7e9d6d74..c269e792 100644 --- a/db/migrate/20250215153930_add_exchange_operating_mic_col_label_to_imports.rb +++ b/db/migrate/20250215153930_add_exchange_operating_mic_col_label_to_imports.rb @@ -1,13 +1,6 @@ class AddExchangeOperatingMicColLabelToImports < ActiveRecord::Migration[7.2] def up add_column :imports, :exchange_operating_mic_col_label, :string - - # Migrate existing trade imports to use the new column - Import.where(type: "TradeImport").find_each do |import| - if import.exchange_col_label.present? - import.update_column(:exchange_operating_mic_col_label, import.exchange_col_label) - end - end end def down diff --git a/db/migrate/20250219084803_remove_exchange_from_import_rows.rb b/db/migrate/20250219084803_remove_exchange_from_import_rows.rb new file mode 100644 index 00000000..15445b30 --- /dev/null +++ b/db/migrate/20250219084803_remove_exchange_from_import_rows.rb @@ -0,0 +1,5 @@ +class RemoveExchangeFromImportRows < ActiveRecord::Migration[7.2] + def change + remove_column :import_rows, :exchange, :string + end +end diff --git a/db/migrate/20250219084839_remove_exchange_col_label_from_imports.rb b/db/migrate/20250219084839_remove_exchange_col_label_from_imports.rb new file mode 100644 index 00000000..533c593e --- /dev/null +++ b/db/migrate/20250219084839_remove_exchange_col_label_from_imports.rb @@ -0,0 +1,5 @@ +class RemoveExchangeColLabelFromImports < ActiveRecord::Migration[7.2] + def change + remove_column :imports, :exchange_col_label, :string + end +end diff --git a/db/migrate/20250219085043_remove_currency_from_securities.rb b/db/migrate/20250219085043_remove_currency_from_securities.rb new file mode 100644 index 00000000..178e8155 --- /dev/null +++ b/db/migrate/20250219085043_remove_currency_from_securities.rb @@ -0,0 +1,6 @@ +class RemoveCurrencyFromSecurities < ActiveRecord::Migration[7.2] + def change + remove_index :securities, :currency + remove_column :securities, :currency, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index 0e92310b..18ba942b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2025_02_15_160141) do +ActiveRecord::Schema[7.2].define(version: 2025_02_19_085043) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -385,7 +385,6 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_15_160141) do t.text "notes" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.string "exchange" t.string "exchange_operating_mic" t.index ["import_id"], name: "index_import_rows_on_import_id" end @@ -418,7 +417,6 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_15_160141) do t.string "error" t.string "currency", default: "USD" t.string "number_format", default: "1,234.56" - t.string "exchange_col_label" t.string "exchange_operating_mic_col_label" t.index ["family_id"], name: "index_imports_on_family_id" end @@ -556,10 +554,8 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_15_160141) do t.string "exchange_mic" t.string "exchange_acronym" t.string "logo_url" - t.string "currency" t.string "exchange_operating_mic" t.index ["country_code"], name: "index_securities_on_country_code" - t.index ["currency"], name: "index_securities_on_currency" t.index ["exchange_operating_mic"], name: "index_securities_on_exchange_operating_mic" t.index ["ticker", "exchange_operating_mic"], name: "index_securities_on_ticker_and_exchange_operating_mic", unique: true end -- 2.53.0 From e53076241bdc11dd10165d6769a480599ed9780c Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Thu, 20 Feb 2025 02:42:39 +0000 Subject: [PATCH 04/12] fix: consolidate import schema and remove redundant columns --- app/models/import/row.rb | 1 - app/models/trade_import.rb | 27 +++++++++---------- ...dd_exchange_operating_mic_to_securities.rb | 17 ++++++++++++ ...xchange_and_currency_columns_to_imports.rb | 5 ---- ...add_exchange_and_currency_to_securities.rb | 6 ----- ...50207062248_add_exchange_to_import_rows.rb | 5 ---- ...0250207194638_adjust_securities_indexes.rb | 6 ----- ...ange_operating_mic_col_label_to_imports.rb | 9 ------- ...d_exchange_operating_mic_to_import_rows.rb | 5 ---- ...084803_remove_exchange_from_import_rows.rb | 5 ---- ..._remove_exchange_col_label_from_imports.rb | 5 ---- ...9085043_remove_currency_from_securities.rb | 6 ----- ...220013452_fix_imports_schema_duplicates.rb | 24 +++++++++++++++++ db/schema.rb | 7 +++-- 14 files changed, 56 insertions(+), 72 deletions(-) delete mode 100644 db/migrate/20250207061453_add_exchange_and_currency_columns_to_imports.rb delete mode 100644 db/migrate/20250207061552_add_exchange_and_currency_to_securities.rb delete mode 100644 db/migrate/20250207062248_add_exchange_to_import_rows.rb delete mode 100644 db/migrate/20250207194638_adjust_securities_indexes.rb delete mode 100644 db/migrate/20250215153930_add_exchange_operating_mic_col_label_to_imports.rb delete mode 100644 db/migrate/20250215160141_add_exchange_operating_mic_to_import_rows.rb delete mode 100644 db/migrate/20250219084803_remove_exchange_from_import_rows.rb delete mode 100644 db/migrate/20250219084839_remove_exchange_col_label_from_imports.rb delete mode 100644 db/migrate/20250219085043_remove_currency_from_securities.rb create mode 100644 db/migrate/20250220013452_fix_imports_schema_duplicates.rb diff --git a/app/models/import/row.rb b/app/models/import/row.rb index b46fe356..d4316a60 100644 --- a/app/models/import/row.rb +++ b/app/models/import/row.rb @@ -3,7 +3,6 @@ class Import::Row < ApplicationRecord validates :amount, numericality: true, allow_blank: true validates :currency, presence: true - validates :exchange_operating_mic, presence: true, if: -> { import.type == "TradeImport" && import.exchange_operating_mic_col_label.present? } validate :date_valid validate :required_columns diff --git a/app/models/trade_import.rb b/app/models/trade_import.rb index 92e89cb7..c73641f2 100644 --- a/app/models/trade_import.rb +++ b/app/models/trade_import.rb @@ -6,10 +6,9 @@ class TradeImport < Import rows.each do |row| account = mappings.accounts.mappable_for(row.account) - # Try to find or create security with exchange validation + # Try to find or create security with ticker only security = find_or_create_security( - ticker: row.ticker, - exchange_operating_mic: row.exchange_operating_mic + ticker: row.ticker ) entry = account.entries.build \ @@ -39,7 +38,7 @@ class TradeImport < Import end def column_keys - %i[date ticker exchange_operating_mic currency qty price account name] + %i[date ticker currency qty price account name] end def dry_run @@ -61,18 +60,17 @@ class TradeImport < Import end private - def find_or_create_security(ticker:, exchange_operating_mic:) + def find_or_create_security(ticker:) # Cache provider responses so that when we're looping through rows and importing, - # we only hit our provider for the unique combinations of ticker / exchange_operating_mic - cache_key = [ ticker, exchange_operating_mic ] + # we only hit our provider for the unique combinations of ticker + cache_key = ticker @provider_securities_cache ||= {} provider_security = @provider_securities_cache[cache_key] ||= begin if Security.security_prices_provider.present? begin response = Security.security_prices_provider.search_securities( - query: ticker, - exchange_operating_mic: exchange_operating_mic + query: ticker ) response.success? ? response.securities.first : nil rescue StandardError => e @@ -82,22 +80,21 @@ class TradeImport < Import end end - if provider_security&.[](:ticker) && provider_security&.[](:exchange_operating_mic) + if provider_security&.[](:ticker) Security.find_or_create_by!( - ticker: provider_security[:ticker], - exchange_operating_mic: provider_security[:exchange_operating_mic] + ticker: provider_security[:ticker] ) do |security| security.name = provider_security[:name] security.country_code = provider_security[:country_code] security.logo_url = provider_security[:logo_url] security.exchange_acronym = provider_security[:exchange_acronym] security.exchange_mic = provider_security[:exchange_mic] + security.exchange_operating_mic = provider_security[:exchange_operating_mic] end else - # If provider data is not available, create security with just the ticker and exchange + # If provider data is not available, create security with just the ticker Security.find_or_create_by!( - ticker: ticker, - exchange_operating_mic: exchange_operating_mic + ticker: ticker ) end end diff --git a/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb b/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb index cdd424f5..51dfaabc 100644 --- a/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb +++ b/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb @@ -1,6 +1,23 @@ class AddExchangeOperatingMicToSecurities < ActiveRecord::Migration[7.2] def change + # Add exchange_operating_mic to securities add_column :securities, :exchange_operating_mic, :string add_index :securities, :exchange_operating_mic + + # Add exchange_operating_mic_col_label to imports + add_column :imports, :exchange_operating_mic_col_label, :string + + # Add exchange_operating_mic to import_rows + add_column :import_rows, :exchange_operating_mic, :string + + # Remove old exchange and currency columns + remove_column :import_rows, :exchange, :string + remove_column :imports, :exchange_col_label, :string + remove_index :securities, :currency if index_exists?(:securities, :currency) + remove_column :securities, :currency, :string + + # Adjust securities indexes + remove_index :securities, name: "index_securities_on_ticker_and_exchange_mic" if index_exists?(:securities, [ :ticker, :exchange_mic ], name: "index_securities_on_ticker_and_exchange_mic") + add_index :securities, [ :ticker, :exchange_operating_mic ], unique: true end end diff --git a/db/migrate/20250207061453_add_exchange_and_currency_columns_to_imports.rb b/db/migrate/20250207061453_add_exchange_and_currency_columns_to_imports.rb deleted file mode 100644 index 76c25119..00000000 --- a/db/migrate/20250207061453_add_exchange_and_currency_columns_to_imports.rb +++ /dev/null @@ -1,5 +0,0 @@ -class AddExchangeAndCurrencyColumnsToImports < ActiveRecord::Migration[7.2] - def change - add_column :imports, :exchange_col_label, :string - end -end diff --git a/db/migrate/20250207061552_add_exchange_and_currency_to_securities.rb b/db/migrate/20250207061552_add_exchange_and_currency_to_securities.rb deleted file mode 100644 index be176e69..00000000 --- a/db/migrate/20250207061552_add_exchange_and_currency_to_securities.rb +++ /dev/null @@ -1,6 +0,0 @@ -class AddExchangeAndCurrencyToSecurities < ActiveRecord::Migration[7.2] - def change - add_column :securities, :currency, :string - add_index :securities, :currency - end -end diff --git a/db/migrate/20250207062248_add_exchange_to_import_rows.rb b/db/migrate/20250207062248_add_exchange_to_import_rows.rb deleted file mode 100644 index ec305185..00000000 --- a/db/migrate/20250207062248_add_exchange_to_import_rows.rb +++ /dev/null @@ -1,5 +0,0 @@ -class AddExchangeToImportRows < ActiveRecord::Migration[7.2] - def change - add_column :import_rows, :exchange, :string - end -end diff --git a/db/migrate/20250207194638_adjust_securities_indexes.rb b/db/migrate/20250207194638_adjust_securities_indexes.rb deleted file mode 100644 index b04d8de1..00000000 --- a/db/migrate/20250207194638_adjust_securities_indexes.rb +++ /dev/null @@ -1,6 +0,0 @@ -class AdjustSecuritiesIndexes < ActiveRecord::Migration[7.2] - def change - remove_index :securities, name: "index_securities_on_ticker_and_exchange_mic" - add_index :securities, [ :ticker, :exchange_operating_mic ], unique: true - end -end diff --git a/db/migrate/20250215153930_add_exchange_operating_mic_col_label_to_imports.rb b/db/migrate/20250215153930_add_exchange_operating_mic_col_label_to_imports.rb deleted file mode 100644 index c269e792..00000000 --- a/db/migrate/20250215153930_add_exchange_operating_mic_col_label_to_imports.rb +++ /dev/null @@ -1,9 +0,0 @@ -class AddExchangeOperatingMicColLabelToImports < ActiveRecord::Migration[7.2] - def up - add_column :imports, :exchange_operating_mic_col_label, :string - end - - def down - remove_column :imports, :exchange_operating_mic_col_label - end -end diff --git a/db/migrate/20250215160141_add_exchange_operating_mic_to_import_rows.rb b/db/migrate/20250215160141_add_exchange_operating_mic_to_import_rows.rb deleted file mode 100644 index 34951bf1..00000000 --- a/db/migrate/20250215160141_add_exchange_operating_mic_to_import_rows.rb +++ /dev/null @@ -1,5 +0,0 @@ -class AddExchangeOperatingMicToImportRows < ActiveRecord::Migration[7.2] - def change - add_column :import_rows, :exchange_operating_mic, :string - end -end diff --git a/db/migrate/20250219084803_remove_exchange_from_import_rows.rb b/db/migrate/20250219084803_remove_exchange_from_import_rows.rb deleted file mode 100644 index 15445b30..00000000 --- a/db/migrate/20250219084803_remove_exchange_from_import_rows.rb +++ /dev/null @@ -1,5 +0,0 @@ -class RemoveExchangeFromImportRows < ActiveRecord::Migration[7.2] - def change - remove_column :import_rows, :exchange, :string - end -end diff --git a/db/migrate/20250219084839_remove_exchange_col_label_from_imports.rb b/db/migrate/20250219084839_remove_exchange_col_label_from_imports.rb deleted file mode 100644 index 533c593e..00000000 --- a/db/migrate/20250219084839_remove_exchange_col_label_from_imports.rb +++ /dev/null @@ -1,5 +0,0 @@ -class RemoveExchangeColLabelFromImports < ActiveRecord::Migration[7.2] - def change - remove_column :imports, :exchange_col_label, :string - end -end diff --git a/db/migrate/20250219085043_remove_currency_from_securities.rb b/db/migrate/20250219085043_remove_currency_from_securities.rb deleted file mode 100644 index 178e8155..00000000 --- a/db/migrate/20250219085043_remove_currency_from_securities.rb +++ /dev/null @@ -1,6 +0,0 @@ -class RemoveCurrencyFromSecurities < ActiveRecord::Migration[7.2] - def change - remove_index :securities, :currency - remove_column :securities, :currency, :string - end -end diff --git a/db/migrate/20250220013452_fix_imports_schema_duplicates.rb b/db/migrate/20250220013452_fix_imports_schema_duplicates.rb new file mode 100644 index 00000000..ae8bdcd1 --- /dev/null +++ b/db/migrate/20250220013452_fix_imports_schema_duplicates.rb @@ -0,0 +1,24 @@ +class FixImportsSchemaDuplicates < ActiveRecord::Migration[7.2] + def up + # First, ensure any existing number_format values are using the default format + execute <<-SQL + UPDATE imports#{' '} + SET number_format = '1,234.56'#{' '} + WHERE number_format IS NULL OR number_format = ''; + SQL + + # Remove the duplicate number_format column (if it exists) and add it back with the default + change_table :imports do |t| + t.remove :number_format if column_exists?(:imports, :number_format) + t.string :number_format, default: '1,234.56' + end + + # Remove the stale currency column if it exists + remove_column :imports, :currency if column_exists?(:imports, :currency) + end + + def down + # No need to restore the duplicate column or stale currency field + raise ActiveRecord::IrreversibleMigration + end +end diff --git a/db/schema.rb b/db/schema.rb index 18ba942b..99147bda 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2025_02_19_085043) do +ActiveRecord::Schema[7.2].define(version: 2025_02_20_013452) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -101,7 +101,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_19_085043) do t.decimal "balance", precision: 19, scale: 4 t.string "currency" t.boolean "is_active", default: true, null: false - t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true + t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true t.uuid "import_id" t.uuid "plaid_account_id" t.boolean "scheduled_for_deletion", default: false @@ -415,9 +415,8 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_19_085043) do t.string "date_format", default: "%m/%d/%Y" t.string "signage_convention", default: "inflows_positive" t.string "error" - t.string "currency", default: "USD" - t.string "number_format", default: "1,234.56" t.string "exchange_operating_mic_col_label" + t.string "number_format", default: "1,234.56" t.index ["family_id"], name: "index_imports_on_family_id" end -- 2.53.0 From 182e7766d94c0b15e4628b2d94b2db988c0e9565 Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Thu, 20 Feb 2025 22:17:57 +0000 Subject: [PATCH 05/12] feat: Enhance trade import with exchange_operating_mic support --- app/models/trade_import.rb | 80 +++++++++-------- ...dd_exchange_operating_mic_to_securities.rb | 23 ----- ...0250207194638_adjust_securities_indexes.rb | 9 ++ ...220013452_fix_imports_schema_duplicates.rb | 24 ------ ...958_update_imports_for_operating_mic_v2.rb | 25 ++++++ db/schema.rb | 4 +- test/fixtures/imports.yml | 5 ++ test/models/trade_import_test.rb | 85 +++++++++++++++++++ 8 files changed, 172 insertions(+), 83 deletions(-) delete mode 100644 db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb create mode 100644 db/migrate/20250207194638_adjust_securities_indexes.rb delete mode 100644 db/migrate/20250220013452_fix_imports_schema_duplicates.rb create mode 100644 db/migrate/20250220153958_update_imports_for_operating_mic_v2.rb create mode 100644 test/models/trade_import_test.rb diff --git a/app/models/trade_import.rb b/app/models/trade_import.rb index c73641f2..f1bbad8d 100644 --- a/app/models/trade_import.rb +++ b/app/models/trade_import.rb @@ -8,7 +8,8 @@ class TradeImport < Import # Try to find or create security with ticker only security = find_or_create_security( - ticker: row.ticker + ticker: row.ticker, + exchange_operating_mic: row.exchange_operating_mic ) entry = account.entries.build \ @@ -38,7 +39,7 @@ class TradeImport < Import end def column_keys - %i[date ticker currency qty price account name] + %i[date ticker exchange_operating_mic currency qty price account name] end def dry_run @@ -60,42 +61,53 @@ class TradeImport < Import end private - def find_or_create_security(ticker:) - # Cache provider responses so that when we're looping through rows and importing, - # we only hit our provider for the unique combinations of ticker - cache_key = ticker + def find_or_create_security(ticker:, exchange_operating_mic:) + # Normalize empty string to nil for consistency + exchange_operating_mic = nil if exchange_operating_mic.blank? + + # First try to find an exact match in our DB + internal_security = Security.find_by(ticker:, exchange_operating_mic:) + return internal_security if internal_security.present? + + # If no exact match and no exchange_operating_mic was provided, try to find any security with the same ticker + if exchange_operating_mic.nil? + internal_security = Security.where(ticker:).first + if internal_security.present? + internal_security.update!(exchange_operating_mic: nil) + return internal_security + end + end + + # If we couldn't find the security locally and the provider isn't available, create with provided info + return Security.create!(ticker: ticker, exchange_operating_mic: exchange_operating_mic) unless Security.security_prices_provider.present? + + # Cache provider responses so that when we're looping through rows and importing, we only hit our provider for the unique combinations of ticker / exchange_operating_mic + cache_key = [ ticker, exchange_operating_mic ] + @provider_securities_cache ||= {} provider_security = @provider_securities_cache[cache_key] ||= begin - if Security.security_prices_provider.present? - begin - response = Security.security_prices_provider.search_securities( - query: ticker - ) - response.success? ? response.securities.first : nil - rescue StandardError => e - Rails.logger.error "Failed to fetch security data: #{e.message}" - nil - end - end + response = Security.security_prices_provider.search_securities( + query: ticker, + exchange_operating_mic: exchange_operating_mic + ) + + return nil unless response.success? + + response.securities.first end - if provider_security&.[](:ticker) - Security.find_or_create_by!( - ticker: provider_security[:ticker] - ) do |security| - security.name = provider_security[:name] - security.country_code = provider_security[:country_code] - security.logo_url = provider_security[:logo_url] - security.exchange_acronym = provider_security[:exchange_acronym] - security.exchange_mic = provider_security[:exchange_mic] - security.exchange_operating_mic = provider_security[:exchange_operating_mic] - end - else - # If provider data is not available, create security with just the ticker - Security.find_or_create_by!( - ticker: ticker - ) - end + return Security.create!(ticker: ticker, exchange_operating_mic: exchange_operating_mic) if provider_security.nil? + + # Create a new security with the provider's data and our exchange_operating_mic + Security.create!( + ticker: ticker, + name: provider_security.dig(:name), + country_code: provider_security.dig(:country_code), + logo_url: provider_security.dig(:logo_url), + exchange_acronym: provider_security.dig(:exchange_acronym), + exchange_mic: provider_security.dig(:exchange_mic), + exchange_operating_mic: exchange_operating_mic + ) end end diff --git a/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb b/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb deleted file mode 100644 index 51dfaabc..00000000 --- a/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb +++ /dev/null @@ -1,23 +0,0 @@ -class AddExchangeOperatingMicToSecurities < ActiveRecord::Migration[7.2] - def change - # Add exchange_operating_mic to securities - add_column :securities, :exchange_operating_mic, :string - add_index :securities, :exchange_operating_mic - - # Add exchange_operating_mic_col_label to imports - add_column :imports, :exchange_operating_mic_col_label, :string - - # Add exchange_operating_mic to import_rows - add_column :import_rows, :exchange_operating_mic, :string - - # Remove old exchange and currency columns - remove_column :import_rows, :exchange, :string - remove_column :imports, :exchange_col_label, :string - remove_index :securities, :currency if index_exists?(:securities, :currency) - remove_column :securities, :currency, :string - - # Adjust securities indexes - remove_index :securities, name: "index_securities_on_ticker_and_exchange_mic" if index_exists?(:securities, [ :ticker, :exchange_mic ], name: "index_securities_on_ticker_and_exchange_mic") - add_index :securities, [ :ticker, :exchange_operating_mic ], unique: true - end -end diff --git a/db/migrate/20250207194638_adjust_securities_indexes.rb b/db/migrate/20250207194638_adjust_securities_indexes.rb new file mode 100644 index 00000000..cc2f1918 --- /dev/null +++ b/db/migrate/20250207194638_adjust_securities_indexes.rb @@ -0,0 +1,9 @@ +class AdjustSecuritiesIndexes < ActiveRecord::Migration[7.2] + def change + # Add back the original index that was removed + add_index :securities, [ :ticker, :exchange_mic ], unique: true, name: "index_securities_on_ticker_and_exchange_mic" + + # Add the new index for exchange_operating_mic + add_index :securities, [ :ticker, :exchange_operating_mic ], unique: true + end +end diff --git a/db/migrate/20250220013452_fix_imports_schema_duplicates.rb b/db/migrate/20250220013452_fix_imports_schema_duplicates.rb deleted file mode 100644 index ae8bdcd1..00000000 --- a/db/migrate/20250220013452_fix_imports_schema_duplicates.rb +++ /dev/null @@ -1,24 +0,0 @@ -class FixImportsSchemaDuplicates < ActiveRecord::Migration[7.2] - def up - # First, ensure any existing number_format values are using the default format - execute <<-SQL - UPDATE imports#{' '} - SET number_format = '1,234.56'#{' '} - WHERE number_format IS NULL OR number_format = ''; - SQL - - # Remove the duplicate number_format column (if it exists) and add it back with the default - change_table :imports do |t| - t.remove :number_format if column_exists?(:imports, :number_format) - t.string :number_format, default: '1,234.56' - end - - # Remove the stale currency column if it exists - remove_column :imports, :currency if column_exists?(:imports, :currency) - end - - def down - # No need to restore the duplicate column or stale currency field - raise ActiveRecord::IrreversibleMigration - end -end diff --git a/db/migrate/20250220153958_update_imports_for_operating_mic_v2.rb b/db/migrate/20250220153958_update_imports_for_operating_mic_v2.rb new file mode 100644 index 00000000..fe62baa0 --- /dev/null +++ b/db/migrate/20250220153958_update_imports_for_operating_mic_v2.rb @@ -0,0 +1,25 @@ +class UpdateImportsForOperatingMicV2 < ActiveRecord::Migration[7.2] + def up + # First remove the old exchange columns if they exist + remove_column :import_rows, :exchange if column_exists?(:import_rows, :exchange) + remove_column :imports, :exchange_col_label if column_exists?(:imports, :exchange_col_label) + + # Then remove and re-add the operating mic columns to ensure they're in the correct state + remove_column :import_rows, :exchange_operating_mic if column_exists?(:import_rows, :exchange_operating_mic) + remove_column :imports, :exchange_operating_mic_col_label if column_exists?(:imports, :exchange_operating_mic_col_label) + + # Add the columns fresh + add_column :import_rows, :exchange_operating_mic, :string + add_column :imports, :exchange_operating_mic_col_label, :string + end + + def down + # Remove the new columns + remove_column :import_rows, :exchange_operating_mic + remove_column :imports, :exchange_operating_mic_col_label + + # Add back the old columns + add_column :import_rows, :exchange, :string + add_column :imports, :exchange_col_label, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index 99147bda..0c4579ab 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.2].define(version: 2025_02_20_013452) do +ActiveRecord::Schema[7.2].define(version: 2025_02_20_153958) do # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" @@ -415,8 +415,8 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_20_013452) do t.string "date_format", default: "%m/%d/%Y" t.string "signage_convention", default: "inflows_positive" t.string "error" - t.string "exchange_operating_mic_col_label" t.string "number_format", default: "1,234.56" + t.string "exchange_operating_mic_col_label" t.index ["family_id"], name: "index_imports_on_family_id" end diff --git a/test/fixtures/imports.yml b/test/fixtures/imports.yml index 97e2cd72..366bb6d9 100644 --- a/test/fixtures/imports.yml +++ b/test/fixtures/imports.yml @@ -2,3 +2,8 @@ transaction: family: dylan_family type: TransactionImport status: pending + +trade: + family: dylan_family + type: TradeImport + status: pending diff --git a/test/models/trade_import_test.rb b/test/models/trade_import_test.rb new file mode 100644 index 00000000..2900034e --- /dev/null +++ b/test/models/trade_import_test.rb @@ -0,0 +1,85 @@ +require "test_helper" +require "ostruct" + +class TradeImportTest < ActiveSupport::TestCase + include ActiveJob::TestHelper, ImportInterfaceTest + + setup do + @subject = @import = imports(:trade) + end + + test "imports trades and accounts" do + # Create an existing AAPL security with no exchange_operating_mic + aapl = Security.create!(ticker: "AAPL", exchange_operating_mic: nil) + + provider = mock + + # We should only hit the provider for GOOGL since AAPL already exists + provider.expects(:search_securities).with( + query: "GOOGL", + exchange_operating_mic: "XNAS" + ).returns( + OpenStruct.new( + securities: [ + { + ticker: "GOOGL", + name: "Google Inc.", + country_code: "US", + exchange_mic: "XNGS", + exchange_operating_mic: "XNAS", + exchange_acronym: "NGS" + } + ], + success?: true, + raw_response: nil + ) + ).once + + Security.stubs(:security_prices_provider).returns(provider) + + import = <<~CSV + date,ticker,qty,price,currency,account,name,exchange_operating_mic + 01/01/2024,AAPL,10,150.00,USD,TestAccount1,Apple Purchase, + 01/02/2024,GOOGL,5,2500.00,USD,TestAccount1,Google Purchase,XNAS + CSV + + @import.update!( + raw_file_str: import, + date_col_label: "date", + ticker_col_label: "ticker", + qty_col_label: "qty", + price_col_label: "price", + exchange_operating_mic_col_label: "exchange_operating_mic", + date_format: "%m/%d/%Y", + signage_convention: "inflows_positive" + ) + + @import.generate_rows_from_csv + + @import.mappings.create! key: "TestAccount1", create_when_empty: true, type: "Import::AccountMapping" + + @import.reload + + assert_difference [ + -> { Account::Entry.count }, + -> { Account::Trade.count } + ], 2 do + assert_difference [ + -> { Security.count }, + -> { Account.count } + ], 1 do + @import.publish + end + end + + assert_equal "complete", @import.status + + # Verify the securities were created/updated correctly + aapl.reload + assert_nil aapl.exchange_operating_mic + + googl = Security.find_by(ticker: "GOOGL") + assert_equal "XNAS", googl.exchange_operating_mic + assert_equal "XNGS", googl.exchange_mic + end +end -- 2.53.0 From 4f8694da92b4551a35b5f34e0845bb26013a2799 Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Fri, 21 Feb 2025 15:11:47 +0000 Subject: [PATCH 06/12] Revert changes to existing migration --- db/migrate/20250207194638_adjust_securities_indexes.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/db/migrate/20250207194638_adjust_securities_indexes.rb b/db/migrate/20250207194638_adjust_securities_indexes.rb index cc2f1918..b04d8de1 100644 --- a/db/migrate/20250207194638_adjust_securities_indexes.rb +++ b/db/migrate/20250207194638_adjust_securities_indexes.rb @@ -1,9 +1,6 @@ class AdjustSecuritiesIndexes < ActiveRecord::Migration[7.2] def change - # Add back the original index that was removed - add_index :securities, [ :ticker, :exchange_mic ], unique: true, name: "index_securities_on_ticker_and_exchange_mic" - - # Add the new index for exchange_operating_mic + remove_index :securities, name: "index_securities_on_ticker_and_exchange_mic" add_index :securities, [ :ticker, :exchange_operating_mic ], unique: true end end -- 2.53.0 From 890f4f6795af0b246bd92b7e728fe75220ec0c99 Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Fri, 21 Feb 2025 15:14:38 +0000 Subject: [PATCH 07/12] Simplify migration to use change method --- ...958_update_imports_for_operating_mic_v2.rb | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/db/migrate/20250220153958_update_imports_for_operating_mic_v2.rb b/db/migrate/20250220153958_update_imports_for_operating_mic_v2.rb index fe62baa0..11f10191 100644 --- a/db/migrate/20250220153958_update_imports_for_operating_mic_v2.rb +++ b/db/migrate/20250220153958_update_imports_for_operating_mic_v2.rb @@ -1,25 +1,6 @@ class UpdateImportsForOperatingMicV2 < ActiveRecord::Migration[7.2] - def up - # First remove the old exchange columns if they exist - remove_column :import_rows, :exchange if column_exists?(:import_rows, :exchange) - remove_column :imports, :exchange_col_label if column_exists?(:imports, :exchange_col_label) - - # Then remove and re-add the operating mic columns to ensure they're in the correct state - remove_column :import_rows, :exchange_operating_mic if column_exists?(:import_rows, :exchange_operating_mic) - remove_column :imports, :exchange_operating_mic_col_label if column_exists?(:imports, :exchange_operating_mic_col_label) - - # Add the columns fresh + def change add_column :import_rows, :exchange_operating_mic, :string add_column :imports, :exchange_operating_mic_col_label, :string end - - def down - # Remove the new columns - remove_column :import_rows, :exchange_operating_mic - remove_column :imports, :exchange_operating_mic_col_label - - # Add back the old columns - add_column :import_rows, :exchange, :string - add_column :imports, :exchange_col_label, :string - end end -- 2.53.0 From 74dbab761bc0a59455f52e2900e52e3d0c03ecc7 Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Fri, 21 Feb 2025 15:17:40 +0000 Subject: [PATCH 08/12] Restore previously deleted migration --- ...250207011850_add_exchange_operating_mic_to_securities.rb | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb diff --git a/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb b/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb new file mode 100644 index 00000000..cdd424f5 --- /dev/null +++ b/db/migrate/20250207011850_add_exchange_operating_mic_to_securities.rb @@ -0,0 +1,6 @@ +class AddExchangeOperatingMicToSecurities < ActiveRecord::Migration[7.2] + def change + add_column :securities, :exchange_operating_mic, :string + add_index :securities, :exchange_operating_mic + end +end -- 2.53.0 From c5d568739c3603f0f0a4ecb248e1a783b37e30f1 Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Fri, 21 Feb 2025 15:20:38 +0000 Subject: [PATCH 09/12] Remove unused import_col_labels method --- app/helpers/imports_helper.rb | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/app/helpers/imports_helper.rb b/app/helpers/imports_helper.rb index 87a58460..67930cc1 100644 --- a/app/helpers/imports_helper.rb +++ b/app/helpers/imports_helper.rb @@ -55,23 +55,6 @@ module ImportsHelper [ base, border ].join(" ") end - def import_col_labels - { - date: "Date", - ticker: "Ticker", - exchange_operating_mic: "Exchange Operating MIC", - currency: "Currency", - qty: "Quantity", - price: "Price", - account: "Account", - name: "Name", - category: "Category", - tags: "Tags", - entity_type: "Entity Type", - notes: "Notes" - } - end - private def permitted_import_types %w[transaction_import trade_import account_import mint_import] -- 2.53.0 From 4a87705d0b28a5ebbe0af93ea296da5a37cabf5e Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Fri, 21 Feb 2025 15:35:59 +0000 Subject: [PATCH 10/12] Update schema.rb after running migrations --- db/schema.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index 0c4579ab..9343a93b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -101,7 +101,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_20_153958) do t.decimal "balance", precision: 19, scale: 4 t.string "currency" t.boolean "is_active", default: true, null: false - t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true + t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true t.uuid "import_id" t.uuid "plaid_account_id" t.boolean "scheduled_for_deletion", default: false -- 2.53.0 From e6871bd63cd8f64cb2a10c19cee736e93435c0ca Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Fri, 21 Feb 2025 17:42:34 +0000 Subject: [PATCH 11/12] Update trade_import.rb and fix schema.rb with db:migrate:reset --- app/models/trade_import.rb | 35 ++++++++++------------------------- db/schema.rb | 4 ++-- 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/app/models/trade_import.rb b/app/models/trade_import.rb index f1bbad8d..6d912fc8 100644 --- a/app/models/trade_import.rb +++ b/app/models/trade_import.rb @@ -65,22 +65,10 @@ class TradeImport < Import # Normalize empty string to nil for consistency exchange_operating_mic = nil if exchange_operating_mic.blank? - # First try to find an exact match in our DB - internal_security = Security.find_by(ticker:, exchange_operating_mic:) + # First try to find an exact match in our DB, or if no exchange_operating_mic is provided, find by ticker only + internal_security = exchange_operating_mic.present? ? Security.find_by(ticker:, exchange_operating_mic:) : Security.find_by(ticker:) return internal_security if internal_security.present? - # If no exact match and no exchange_operating_mic was provided, try to find any security with the same ticker - if exchange_operating_mic.nil? - internal_security = Security.where(ticker:).first - if internal_security.present? - internal_security.update!(exchange_operating_mic: nil) - return internal_security - end - end - - # If we couldn't find the security locally and the provider isn't available, create with provided info - return Security.create!(ticker: ticker, exchange_operating_mic: exchange_operating_mic) unless Security.security_prices_provider.present? - # Cache provider responses so that when we're looping through rows and importing, we only hit our provider for the unique combinations of ticker / exchange_operating_mic cache_key = [ ticker, exchange_operating_mic ] @@ -97,17 +85,14 @@ class TradeImport < Import response.securities.first end - return Security.create!(ticker: ticker, exchange_operating_mic: exchange_operating_mic) if provider_security.nil? + return Security.find_or_create_by!(ticker: ticker, exchange_operating_mic: nil) if provider_security.nil? - # Create a new security with the provider's data and our exchange_operating_mic - Security.create!( - ticker: ticker, - name: provider_security.dig(:name), - country_code: provider_security.dig(:country_code), - logo_url: provider_security.dig(:logo_url), - exchange_acronym: provider_security.dig(:exchange_acronym), - exchange_mic: provider_security.dig(:exchange_mic), - exchange_operating_mic: exchange_operating_mic - ) + Security.find_or_create_by!(ticker: provider_security.dig(:ticker), exchange_operating_mic: provider_security.dig(:exchange_operating_mic)) do |security| + security.name = provider_security.dig(:name) + security.country_code = provider_security.dig(:country_code) + security.logo_url = provider_security.dig(:logo_url) + security.exchange_acronym = provider_security.dig(:exchange_acronym) + security.exchange_mic = provider_security.dig(:exchange_mic) + end end end diff --git a/db/schema.rb b/db/schema.rb index 484199ea..ccbe329a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -101,7 +101,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_20_153958) do t.decimal "balance", precision: 19, scale: 4 t.string "currency" t.boolean "is_active", default: true, null: false - t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY (ARRAY[('Loan'::character varying)::text, ('CreditCard'::character varying)::text, ('OtherLiability'::character varying)::text])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true + t.virtual "classification", type: :string, as: "\nCASE\n WHEN ((accountable_type)::text = ANY ((ARRAY['Loan'::character varying, 'CreditCard'::character varying, 'OtherLiability'::character varying])::text[])) THEN 'liability'::text\n ELSE 'asset'::text\nEND", stored: true t.uuid "import_id" t.uuid "plaid_account_id" t.boolean "scheduled_for_deletion", default: false @@ -415,7 +415,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_02_20_153958) do t.string "date_format", default: "%m/%d/%Y" t.string "signage_convention", default: "inflows_positive" t.string "error" - t.string "number_format", default: "1,234.56" + t.string "number_format" t.string "exchange_operating_mic_col_label" t.index ["family_id"], name: "index_imports_on_family_id" end -- 2.53.0 From 423c2f9a0d29503d6e6d4eb5603fda046679ad44 Mon Sep 17 00:00:00 2001 From: David Anyatonwu Date: Fri, 21 Feb 2025 23:27:22 +0000 Subject: [PATCH 12/12] fix: improve trade import security creation --- app/models/trade_import.rb | 41 ++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/app/models/trade_import.rb b/app/models/trade_import.rb index 6d912fc8..ddaad904 100644 --- a/app/models/trade_import.rb +++ b/app/models/trade_import.rb @@ -66,33 +66,48 @@ class TradeImport < Import exchange_operating_mic = nil if exchange_operating_mic.blank? # First try to find an exact match in our DB, or if no exchange_operating_mic is provided, find by ticker only - internal_security = exchange_operating_mic.present? ? Security.find_by(ticker:, exchange_operating_mic:) : Security.find_by(ticker:) + internal_security = if exchange_operating_mic.present? + Security.find_by(ticker:, exchange_operating_mic:) + else + Security.find_by(ticker:) + end + return internal_security if internal_security.present? - # Cache provider responses so that when we're looping through rows and importing, we only hit our provider for the unique combinations of ticker / exchange_operating_mic - cache_key = [ ticker, exchange_operating_mic ] + # If security prices provider isn't properly configured or available, create with nil exchange_operating_mic + provider = Security.security_prices_provider + unless provider.present? && provider.respond_to?(:search_securities) + return Security.find_or_create_by!(ticker: ticker, exchange_operating_mic: nil) + end + # Cache provider responses so that when we're looping through rows and importing, + # we only hit our provider for the unique combinations of ticker / exchange_operating_mic + cache_key = [ ticker, exchange_operating_mic ] @provider_securities_cache ||= {} provider_security = @provider_securities_cache[cache_key] ||= begin - response = Security.security_prices_provider.search_securities( + response = provider.search_securities( query: ticker, exchange_operating_mic: exchange_operating_mic ) - return nil unless response.success? - - response.securities.first + if !response || !response.success? || !response.securities || response.securities.empty? + nil + else + response.securities.first + end + rescue => e + nil end return Security.find_or_create_by!(ticker: ticker, exchange_operating_mic: nil) if provider_security.nil? - Security.find_or_create_by!(ticker: provider_security.dig(:ticker), exchange_operating_mic: provider_security.dig(:exchange_operating_mic)) do |security| - security.name = provider_security.dig(:name) - security.country_code = provider_security.dig(:country_code) - security.logo_url = provider_security.dig(:logo_url) - security.exchange_acronym = provider_security.dig(:exchange_acronym) - security.exchange_mic = provider_security.dig(:exchange_mic) + Security.find_or_create_by!(ticker: provider_security[:ticker], exchange_operating_mic: provider_security[:exchange_operating_mic]) do |security| + security.name = provider_security[:name] + security.country_code = provider_security[:country_code] + security.logo_url = provider_security[:logo_url] + security.exchange_acronym = provider_security[:exchange_acronym] + security.exchange_mic = provider_security[:exchange_mic] end end end -- 2.53.0