Use activerecord-import for bulk importing tracepoint records

Non-rigourous testing shows a significant speedup, even on ssds.

diff --git a/Gemfile b/Gemfile
index 372b7d021740b2cecc5949a33d7122b42d0c532f..83f298101fcd38c2459a8ce25087333864c376d7 100644 (file)
--- a/Gemfile
+++ b/Gemfile
@@ -46,6 +46,7 @@ gem "image_optim_rails"
 # Load rails plugins
 gem "actionpack-page_caching"
 gem "active_record_union"
 # Load rails plugins
 gem "actionpack-page_caching"
 gem "active_record_union"

+gem "activerecord-import"

 gem "cancancan"
 gem "composite_primary_keys", "~> 11.1.0"
 gem "config"
 gem "cancancan"
 gem "composite_primary_keys", "~> 11.1.0"
 gem "config"

diff --git a/Gemfile.lock b/Gemfile.lock
index 2b5cc1dd5a6538be1095af7342c0d86e6436dfdb..57feba1fd0c4623ab2f34d99ed4f3f4c5d2ea0e3 100644 (file)
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -40,6 +40,8 @@ GEM
       activemodel (= 5.2.2.1)
       activesupport (= 5.2.2.1)
       arel (>= 9.0)
       activemodel (= 5.2.2.1)
       activesupport (= 5.2.2.1)
       arel (>= 9.0)

+    activerecord-import (0.28.1)
+      activerecord (>= 3.2)

     activestorage (5.2.2.1)
       actionpack (= 5.2.2.1)
       activerecord (= 5.2.2.1)
     activestorage (5.2.2.1)
       actionpack (= 5.2.2.1)
       activerecord (= 5.2.2.1)


@@ -423,6 +425,7 @@ DEPENDENCIES
   aasm
   actionpack-page_caching
   active_record_union
   aasm
   actionpack-page_caching
   active_record_union

+  activerecord-import

   annotate
   autoprefixer-rails (~> 8.6.3)
   better_errors
   annotate
   autoprefixer-rails (~> 8.6.3)
   better_errors

diff --git a/app/models/trace.rb b/app/models/trace.rb
index 892d41a2a217b5872cacc8a5724f63a36da5690c..bd8ab72b3cfe340eb62ed068a8ac6068cab20fed 100644 (file)
--- a/app/models/trace.rb
+++ b/app/models/trace.rb
@@ -289,6 +289,8 @@ class Trace < ActiveRecord::Base
     # If there are any existing points for this trace then delete them
     Tracepoint.where(:gpx_id => id).delete_all
 
     # If there are any existing points for this trace then delete them
     Tracepoint.where(:gpx_id => id).delete_all
 

+    # Gather the trace points together for a bulk import
+    tracepoints = []

     gpx.points do |point|
       if first
         f_lat = point.latitude
     gpx.points do |point|
       if first
         f_lat = point.latitude


@@ -303,9 +305,16 @@ class Trace < ActiveRecord::Base
       tp.timestamp = point.timestamp
       tp.gpx_id = id
       tp.trackid = point.segment
       tp.timestamp = point.timestamp
       tp.gpx_id = id
       tp.trackid = point.segment

-      tp.save!
+      tracepoints << tp

     end
 
     end
 

+    # Run the before_save and before_create callbacks, and then import them in bulk with activerecord-import
+    tracepoints.each do |tp|
+      tp.run_callbacks(:save) { false }
+      tp.run_callbacks(:create) { false }
+    end
+    Tracepoint.import(tracepoints)
+

     if gpx.actual_points.positive?
       max_lat = Tracepoint.where(:gpx_id => id).maximum(:latitude)
       min_lat = Tracepoint.where(:gpx_id => id).minimum(:latitude)
     if gpx.actual_points.positive?
       max_lat = Tracepoint.where(:gpx_id => id).maximum(:latitude)
       min_lat = Tracepoint.where(:gpx_id => id).minimum(:latitude)

diff --git a/test/models/trace_test.rb b/test/models/trace_test.rb
index f03488a6115324e631624edf397f461d1960f8ed..81120f0e776e0c1103560f153f50912eed50d096 100644 (file)
--- a/test/models/trace_test.rb
+++ b/test/models/trace_test.rb
@@ -206,6 +206,10 @@ class TraceTest < ActiveSupport::TestCase
 
       trace.reload
       assert_equal 1, Tracepoint.where(:gpx_id => trace.id).count
 
       trace.reload
       assert_equal 1, Tracepoint.where(:gpx_id => trace.id).count

+
+      # Check that the tile has been set prior to the bulk import
+      # i.e. that the callbacks have been run correctly
+      assert_equal 3221331576, Tracepoint.where(:gpx_id => trace.id).first.tile

     end
   end
 
     end
   end