ignore visited objects

1 files changed, 23 insertions, 12 deletions

diff --git a/helpers.rb b/helpers.rb
index a43966b..41dcc33 100644
--- a/helpers.rb
+++ b/helpers.rb
@@ -1,7 +1,4 @@
-# frozen_string_literal: true
-
 require 'English'
-
 helpers do
   # add date and id, save
   def save_activity(activity, box)
@@ -15,9 +12,9 @@ helpers do
 
       activity['id'] = File.join(box[:url], activity_rel_path)
       activity['object']['published'] = date unless activity['object'].is_a? String
+      # save object
+      save_object activity['object'], box if %w[Create Announce Update].include? activity['type']
     end
-    # save object
-    save_object activity['object'], box if %w[Create Announce Update].include? activity['type']
     # save activity
     FileUtils.mkdir_p File.dirname(activity_path)
     File.open(activity_path, 'w+') { |f| f.puts activity.to_json }
@@ -26,15 +23,18 @@ helpers do
 
   def save_object(object, box)
     object = fetch(object) if object.is_a? String and object.match(/^http/)
-    return unless object # and object['type'] != 'Person'
+    return unless object and object['type'] != 'Person'
+    # File.open(File.join(INBOX[:dir]), 'visited', 'w+').open { |f| f.puts object['id'] }
+    return if box == INBOX and object['id'] and File.readlines(File.join(INBOX[:dir], 'visited'),
+                                                               chomp: true).include? object['id']
 
-    unless object['attributedTo']
+    object['@context'] = 'https://www.w3.org/ns/activitystreams'
+    if object['attributedTo']
+      basename = "#{object['published']}_#{mention(object['attributedTo'])}.json"
+    else
+      basename = "#{object['published']}.json"
       jj object
-      return
     end
-
-    object['@context'] = 'https://www.w3.org/ns/activitystreams'
-    basename = "#{object['published']}_#{mention(object['attributedTo'])}.json"
     object_rel_path = File.join 'object', object['type'].downcase, basename
     object['id'] ||= File.join box[:url], object_rel_path # if box == OUTBOX
     object_path = File.join box[:dir], object_rel_path
@@ -62,6 +62,8 @@ helpers do
           f.puts tag_collection.to_json
         end
       end
+    elsif box == INBOX
+      File.open(File.join(INBOX[:dir], 'visited'), 'a+') { |f| f.puts object['id'] }
     end
     object
   end
@@ -159,7 +161,7 @@ helpers do
 
   def media_type(url) # TODO: extend extensions
     extensions = {
-      image: %w[jpeg jpg png tiff],
+      image: %w[jpeg jpg png tiff webp],
       audio: %w[flac wav mp3 ogg aiff],
       video: %w[mp4 webm]
     }
@@ -173,4 +175,13 @@ helpers do
       JSON.load_file(f)['id'] == id
     end
   end
+
+  def find_id(id, return_filename = true)
+    Dir[File.join('**', '*.json')].find do |f|
+      content = JSON.load_file(f)
+      if content['id'] == id
+        return_filename ? f : content
+      end
+    end
+  end
 end