Changes for NTU Student Affairs

lib/rss_ntu_job.rb

@@ -0,0 +1,99 @@
+# encoding: utf-8
+
+# encoding: utf-8
+
+require 'rss'
+require 'mongo'
+
+AUTHORS = %w[ 學生事務處-學生住宿服務組
+            學生事務處-學生職業生涯發展中心
+            學生事務處-衛生保健及醫療中心
+            學生事務處-課外活動指導組
+            學生事務處-學生活動中心管理組
+            學生事務處-僑生及陸生輔導組
+            學生事務處-生活輔導組
+            學生事務處-軍訓組
+            學生事務處-學生心理輔導中心
+            學生事務處-學生事務處]
+
+DB_BASE_NAME = "site"
+
+all = []
+continue = true
+i = 1
+yesterday = Time.now - 86400
+
+while continue do
+  open("http://ann.cc.ntu.edu.tw/asp/rss.asp?page=#{i}") do |rss|
+    feed = RSS::Parser.parse(rss.read.encode('utf-8', 'big5', invalid: :replace, undef: :replace, replace: '').gsub('<pubDate>Wes,', '<pubDate>Wed,').gsub(/(encoding=\"big5\")/, 'encoding="utf-8"'))
+    feed.items.each do |item|
+      if item.pubDate > yesterday
+        if AUTHORS.include?(item.author)
+          author = item.author.strip
+          category = item.category.to_s.gsub(/\<(\/)*category\>/, '')
+          all << {title: item.title.strip, author: author, link: item.link.strip, date: item.pubDate, category: category, description: item.description.gsub("\r\n", '<br/>').strip}
+        end
+      else
+        continue = false
+        break
+      end
+    end
+  end
+  i += 1
+end
+
+# Get corresponding category_id or create a new one
+def get_category_id(category, categories, coll_cat)
+  if categories.keys.include? "rss_#{category}"
+    [categories["rss_#{category}"], categories]
+  else
+    cat = {
+      _type: "BulletinCategory",
+      key: "rss_#{category}",
+      disable: false,
+      title: {:zh_tw => category},
+      created_at: Time.now,
+      updated_at: Time.now
+    }
+    categories["rss_#{category}"] = result = coll_cat.save(cat)
+    [result, categories]
+  end
+end
+
+# Get categories and id based on a given site number
+def get_mongo_and_categories(site_number="0")
+  db = Mongo::Connection.new("localhost", 27017).db("#{DB_BASE_NAME}_#{site_number}")
+  coll_bulletin = db["bulletins"]
+  coll_cat =  db["bulletin_categories"]
+
+  categories = coll_cat.find().to_a.inject({}) do |categories, category|
+    categories[category['key']] = category['_id']
+    categories
+  end
+  [categories, coll_bulletin, coll_cat]
+end
+
+# Get categories
+categories, coll_bulletin, coll_cat = get_mongo_and_categories
+
+all.each do |bul| # Loop through all the items
+  category_id, categories = get_category_id(bul[:category], categories, coll_cat)
+  unless coll_bulletin.find_one(rss_link: bul[:link]) 
+    bulletin = {  _type: "Bulletin",
+                  postdate: bul[:date],
+                  created_at: bul[:date],
+                  updated_at: bul[:date],
+                  is_checked: true,
+                  is_pending: false,
+                  is_rejected: false,
+                  bulletin_category_id: category_id,
+                  title: {:zh_tw => bul[:title]},
+                  text: {:zh_tw => bul[:description]},
+                  available_for_zh_tw: true,
+                  rss_link: bul[:link] }
+    coll_bulletin.save(bulletin) 
+  end
+end
+
+
+

lib/rss_ntu_migrate.rb

@@ -0,0 +1,91 @@
+# encoding: utf-8
+
+require 'rss'
+require 'mongo'
+
+AUTHORS = %w[ 學生事務處-學生住宿服務組
+            學生事務處-學生職業生涯發展中心
+            學生事務處-衛生保健及醫療中心
+            學生事務處-課外活動指導組
+            學生事務處-學生活動中心管理組
+            學生事務處-僑生及陸生輔導組
+            學生事務處-生活輔導組
+            學生事務處-軍訓組
+            學生事務處-學生心理輔導中心
+            學生事務處-學生事務處]
+
+DB_BASE_NAME = "site"
+
+all = []
+continue = true
+i = 1
+while continue do
+  open("http://ann.cc.ntu.edu.tw/asp/rss.asp?page=#{i}") do |rss|
+    feed = RSS::Parser.parse(rss.read.encode('utf-8', 'big5', invalid: :replace, undef: :replace, replace: '').gsub('<pubDate>Wes,', '<pubDate>Wed,').gsub(/(encoding=\"big5\")/, 'encoding="utf-8"'))
+    feed.items.size
+    feed.items.each do |item|
+      if AUTHORS.include?(item.author)
+        author = item.author.strip
+        category = item.category.to_s.gsub(/\<(\/)*category\>/, '')
+        all << {title: item.title.strip, author: author, link: item.link.strip, date: item.pubDate, category: category, description: item.description.gsub("\r\n", '<br/>').strip}
+      end
+    end
+    continue = false if feed.items.size < 100
+  end
+  i += 1
+end
+
+# Get corresponding category_id or create a new one
+def get_category_id(category, categories, coll_cat)
+  if categories.keys.include? "rss_#{category}"
+    [categories["rss_#{category}"], categories]
+  else
+    cat = {
+      _type: "BulletinCategory",
+      key: "rss_#{category}",
+      disable: false,
+      title: {:zh_tw => category},
+      created_at: Time.now,
+      updated_at: Time.now
+    }
+    categories["rss_#{category}"] = result = coll_cat.save(cat)
+    [result, categories]
+  end
+end
+
+# Get categories and id based on a given site number
+def get_mongo_and_categories(site_number="0")
+  db = Mongo::Connection.new("localhost", 27017).db("#{DB_BASE_NAME}_#{site_number}")
+  coll_bulletin = db["bulletins"]
+  coll_cat =  db["bulletin_categories"]
+
+  categories = coll_cat.find().to_a.inject({}) do |categories, category|
+    categories[category['key']] = category['_id']
+    categories
+  end
+  [categories, coll_bulletin, coll_cat]
+end
+
+# Get categories
+categories, coll_bulletin, coll_cat = get_mongo_and_categories
+
+all.each do |bul| # Loop through all the items
+  category_id, categories = get_category_id(bul[:category], categories, coll_cat)
+  unless coll_bulletin.find_one(rss_link: bul[:link]) 
+    bulletin = {  _type: "Bulletin",
+                  postdate: bul[:date],
+                  created_at: bul[:date],
+                  updated_at: bul[:date],
+                  is_checked: true,
+                  is_pending: false,
+                  is_rejected: false,
+                  bulletin_category_id: category_id,
+                  title: {:zh_tw => bul[:title]},
+                  text: {:zh_tw => bul[:description]},
+                  available_for_zh_tw: true,
+                  rss_link: bul[:link] }
+    coll_bulletin.save(bulletin) 
+  end
+end
+
+

lib/tasks/ntu_mongo_files.rake

@@ -26,18 +26,12 @@ namespace :mongo_files do
 
   IMAGE_UPLOADER_MODELS = %w[ad_image bulletin image site]
 
-  # ad_banner
-  # bulletin
-  # design
-  # site
-
   task :clean => :environment do
     @files = Mongoid.database['fs.files']
     @chunks = Mongoid.database['fs.chunks']
     clean_duplicates
     clean_unused
     remove_objects
-    # remove_unlinked
   end
 
   # Remove unused files from db
@@ -81,7 +75,7 @@ namespace :mongo_files do
   # Remove duplicate files from db
   def clean_duplicates
     p "# of files in db: #{@files.count}"
-    file_hash = @files.find().inject({}) do |db_files, file|
+    file_hash = @files.find().sort({uploadDate: -1}).inject({}) do |db_files, file|
       name = file['filename']
       if db_files[name]
         db_files[name] += [file['_id']]
@@ -119,17 +113,8 @@ namespace :mongo_files do
     Mongoid.database['writing_journals'].remove()
     Mongoid.database['writing_journal_files'].remove()
 
-    # Destroy bulletins, bulletin_files and bulletin_links
-    Bulletin.excludes(create_user_id: nil).destroy
-    BulletinFile.destroy_all
-    BulletinLink.destroy_all
-
-    # Destroy gallery_albums and gallery_images
-    Mongoid.database['gallery_albums'].remove()
-    Mongoid.database['gallery_images'].remove()
-
     db_array = @files.find().inject([]) do |db_files, file|
-      db_files << file['filename'] if file['filename'] =~ /writing_journal_file|news_bulletin_file|bulletin_file|gallery_image|image\/image/
+      db_files << file['filename'] if file['filename'] =~ /writing_journal_file|news_bulletin_file|image\/image/
       db_files
     end
     p db_array.size