Changes for NTU General Affairs

lib/ntu_archive/archive_ntu_dd.rb

@@ -0,0 +1,144 @@
+# encoding : utf-8
+
+# Usage ===============
+#
+# ARGV[0] = folder path of xml files
+# ARGV[1] = folder path of files
+#
+#======================
+
+
+require 'mongo'
+require 'nokogiri'
+include Mongo
+
+CSV_PATH = ARGV[0]
+FILE_PATH = ARGV[1]
+FIELDS = %w[title file createdate modifydate createuser modifyuser]
+DB_BASE_NAME = "production"
+
+TABLE = {
+  "0" => ["gaTaco", "ntuga"],
+  "1" => ["fdLai", "fdhome"],
+  "5" => ["prLin", "property01"],
+  "3" => ["gsTien", "ckh", "general01", "lynn"],
+  "10" => ["cmChen", "construction01", "allen", "enAmeliaxu", "energy"],
+#  "2" => ["caLin", "cashier01"],
+  "6" => ["pcAmyok", "purchasing01", "purchasing02"],
+  "7" => ["fmHsyu", "ntufss"],
+  "4" => ["soChang", "social01"],
+  "8" => ["meJune", "medicine01"],
+  "9" => ["plKoa", "police"]
+}
+
+which_site = {}
+
+data = []
+
+Dir.foreach(ARGV[0]) do |file|
+  if file =~ /\w\.\w/
+    doc = Nokogiri::XML(File.open(File.join(ARGV[0], file)))
+    doc.xpath("//row").each { |row| 
+      xml_fields = row.children.map{ |row| [row.name, row.content] }
+      fields = FIELDS.inject([]) do |fields, field|
+        xml_fields.each do |xf|
+          if xf[0].include?(field)
+            fields << xf[1] 
+            break
+          end
+        end
+        fields
+      end
+      data << fields + [File.basename(file, '.xml')]
+    }
+    data.each do |d|
+      title, file, c_time, m_time, creator, modifier, category = d
+      if file =~ /\w\.\w/
+        file = File.join(File.join(ARGV[1], file))
+        site = ""
+        TABLE.each do |key, value|
+          if value.include? modifier
+            site = key
+            break
+          elsif value.include? creator
+            site = key
+            break
+          end
+        end
+        site = "0" if site.empty?
+        if which_site[site]
+          if which_site[site][category]
+            which_site[site][category] += [[title, file, c_time, m_time]]
+          else
+            which_site[site][category] = [[title, file, c_time, m_time]]
+          end
+        else
+          which_site[site] = {category => [[title, file, c_time, m_time]]}
+        end
+      end
+    end
+  end
+end
+
+def save_category archive_file_category, in_category
+  category = {
+    title: {"zh_tw" => in_category, "en" => in_category},
+    key: "import_#{category}"
+  }
+  archive_file_category.save(category)
+end
+
+def save_file_multiples archive_file_multiples, grid ,archive_id,title, file, c_time, m_time
+  a_file = File.open(file) rescue nil
+  if a_file
+    filename = File.basename(file)
+    archive_file = {
+      file_title: {"zh_tw" => title, "en" => title },
+      choose_lang: ["zh_tw", "en"],
+      file: filename,
+      archive_file_id: archive_id,
+      created_at: c_time,
+      updated_at: m_time
+    }
+    a_file_id = archive_file_multiples.save(archive_file)
+    grid.put(a_file, filename: "assets/archive_file_multiple/file/#{a_file_id}/#{filename}")
+  end
+end
+
+def save_archive archive_files, category_id
+  archive = {
+    title: {"zh_tw" => " ", "en" => " "},
+    archive_file_category_id: category_id,
+    is_top: false,
+    is_hot: false,
+    is_hidden: false,
+    created_at: c_time,
+    updated_at: m_time
+  }
+  archive_files.save(archive)
+end
+
+which_site.each do |site, categories|
+
+  db = MongoClient.new("localhost", 27017).db("#{DB_BASE_NAME}_#{site}")
+
+  categories.each do |category, data|
+    archive_file_category = db.collection('archive_file_categories')
+    category_id = save_category archive_file_category, category
+
+    archive_files = db.collection('archive_files')
+    archive_id = save_archive archive_files, category_id
+
+    grid = Grid.new(db)
+    archive_file_multiples = db.collection('archive_file_multiples')
+    file_ids = []
+
+    data.each do |file|
+      save_file_multiples(archive_file_multiples, grid ,archive_id , *file)
+    end
+  end
+
+end
+
+
+

lib/rss_ntu_0_to_11.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+
+require 'mongo'
+include Mongo
+
+DB_BASE_NAME = "production"
+
+main_db = MongoClient.new("localhost", 27017).db("#{DB_BASE_NAME}_#{0}")
+main_bulletin = main_db["bulletins"]
+main_cat =  main_db["bulletin_categories"]
+
+copy_db = MongoClient.new("localhost", 27017).db("#{DB_BASE_NAME}_#{11}")
+copy_bulletin = copy_db["bulletins"]
+copy_cat =  copy_db["bulletin_categories"]
+
+
+main_cat.find(key: /rss_/).each do |category|
+  category_id = \
+  if cat = copy_cat.find_one(key: category['key'])
+    cat['_id']
+  else
+    copy_category = category.clone
+    copy_category['_id'] = BSON::ObjectId.new
+    copy_cat.save copy_category
+  end
+
+  main_bulletin.find(bulletin_category_id: category['_id']).each do |bulletin|
+    unless copy_bulletin.find_one(rss_link: bulletin['rss_link'])
+      copybulletin = bulletin.clone
+      copybulletin['_id'] = BSON::ObjectId.new
+      copybulletin['bulletin_category_id'] = category_id
+      bulletin_id = copy_bulletin.save copybulletin
+    end
+  end
+end

lib/rss_ntu_job.rb

@@ -0,0 +1,129 @@
+# encoding: utf-8
+
+require 'rss'
+require 'mongo'
+
+SITES = { "總務處-各單位公告" => "0",
+          "總務處-文書組" => "1",
+          "總務處-出納組" => "2",
+          "總務處-事務組" => "3",
+          "總務處-保管組" => "5",
+          "總務處-採購組" => "6",
+          "總務處-經營管理組" => "7",
+          "總務處-駐衛警察隊" => "9",
+          "總務處-營繕組" => "10",
+          "總務處-總務處" => "11",
+          "社會科學院-社會科學院總務分處" => "4",
+          "醫學院-醫學院總務分處" => "8" }
+
+SITE_KEYS = SITES.keys
+
+DB_BASE_NAME = "production"
+
+all = {}
+continue = true
+i = 1
+yesterday = Time.now - 86400
+
+while continue do
+  open("http://ann.cc.ntu.edu.tw/asp/rss.asp?page=#{i}") do |rss|
+    feed = RSS::Parser.parse(rss.read.encode('utf-8', 'big5', invalid: :replace, undef: :replace, replace: '').gsub('<pubDate>Wes,', '<pubDate>Wed,').gsub(/(encoding=\"big5\")/, 'encoding="utf-8"'))
+    feed.items.each do |item|
+      if item.pubDate > yesterday
+        if SITE_KEYS.include?(item.author)
+          author = item.author.strip
+          category = item.category.to_s.gsub(/\<(\/)*category\>/, '')
+          if all[author]
+            all[author][item.link.strip] = {title: item.title.strip, author: author, link: item.link.strip, date: item.pubDate, category: category, description: item.description.gsub("\r\n", '<br/>').strip}
+          else
+            all[author] = {item.link.strip => {title: item.title.strip, author: author, link: item.link.strip, date: item.pubDate, category: category, description: item.description.gsub("\r\n", '<br/>').strip}}
+          end
+        end
+      else
+        continue = false
+        break
+      end
+    end
+  end
+  i += 1
+end
+
+# Get corresponding category_id or create a new one
+def get_category_id(category, categories, coll_cat)
+  if categories.keys.include? "rss_#{category}"
+    [categories["rss_#{category}"], categories]
+  else
+    cat = {
+      _type: "BulletinCategory",
+      key: "rss_#{category}",
+      disable: false,
+      title: {:zh_tw => category},
+      created_at: Time.now,
+      updated_at: Time.now
+    }
+    categories["rss_#{category}"] = result = coll_cat.save(cat)
+    [result, categories]
+  end
+end
+
+# Get categories and id based on a given site number
+def get_mongo_and_categories(site_number="0")
+  db = Mongo::Connection.new("localhost", 27017).db("#{DB_BASE_NAME}_#{site_number}")
+  coll_bulletin = db["bulletins"]
+  coll_cat =  db["bulletin_categories"]
+
+  categories = coll_cat.find().to_a.inject({}) do |categories, category|
+    categories[category['key']] = category['_id']
+    categories
+  end
+  [categories, coll_bulletin, coll_cat]
+end
+
+# Get main site (總務處) categories
+@main_categories, @main_coll_bulletin, @main_coll_cat = get_mongo_and_categories
+@copy_categories, @copy_coll_bulletin, @copy_coll_cat = get_mongo_and_categories('11')
+
+all.each do |key, value| # Loop through all the authors
+  site_number = SITES[key]
+  categories, coll_bulletin, coll_cat = get_mongo_and_categories(site_number) # Get current's site categories
+  value.each_value do |bul| # Loop through all the items
+    category_id, categories = get_category_id(bul[:category], categories, coll_cat)
+    unless coll_bulletin.find_one(rss_link: bul[:link]) 
+      bulletin = {  _type: "Bulletin",
+                    postdate: bul[:date],
+                    created_at: bul[:date],
+                    updated_at: bul[:date],
+                    is_checked: true,
+                    is_pending: false,
+                    is_rejected: false,
+                    bulletin_category_id: category_id,
+                    title: {:zh_tw => bul[:title]},
+                    text: {:zh_tw => bul[:description]},
+                    available_for_zh_tw: true,
+                    rss_link: bul[:link],
+                    is_top: false,
+                    is_hot: false,
+                    is_hidden: false }
+      coll_bulletin.save(bulletin) 
+
+      unless site_number.eql?("0") || @main_coll_bulletin.find_one(rss_link: bul[:link]) # Copy the item to the main site
+
+        category_id, @main_categories = get_category_id(bul[:category], @main_categories, @main_coll_cat)
+        main_bulletin = bulletin.clone
+        main_bulletin['_id'] = BSON::ObjectId.new
+        main_bulletin[:bulletin_category_id] = category_id
+        @main_coll_bulletin.save(main_bulletin)
+
+        category_id, @copy_categories = get_category_id(bul[:category], @copy_categories, @copy_coll_cat)
+        copy_bulletin = bulletin.clone
+        copy_bulletin['_id'] = BSON::ObjectId.new
+        copy_bulletin[:bulletin_category_id] = category_id
+        @copy_coll_bulletin.save(copy_bulletin)
+
+      end
+    end
+  end
+end
+
+
+

lib/rss_ntu_migration.rb

@@ -0,0 +1,113 @@
+# encoding: utf-8
+
+require 'rss'
+require 'mongo'
+
+SITES = { "總務處-各單位公告" => "0",
+          "總務處-文書組" => "1",
+          "總務處-出納組" => "2",
+          "總務處-事務組" => "3",
+          "總務處-保管組" => "5",
+          "總務處-採購組" => "6",
+          "總務處-經營管理組" => "7",
+          "總務處-駐衛警察隊" => "9",
+          "總務處-營繕組" => "10",
+          "總務處-總務處" => "11",
+          "社會科學院-社會科學院總務分處" => "4",
+          "醫學院-醫學院總務分處" => "8" }
+
+SITE_KEYS = SITES.keys
+
+DB_BASE_NAME = "production"
+
+all = {}
+continue = true
+i = 1
+while continue do
+  open("http://ann.cc.ntu.edu.tw/asp/rss.asp?page=#{i}") do |rss|
+    feed = RSS::Parser.parse(rss.read.encode('utf-8', 'big5', invalid: :replace, undef: :replace, replace: '').gsub('<pubDate>Wes,', '<pubDate>Wed,').gsub(/(encoding=\"big5\")/, 'encoding="utf-8"'))
+    feed.items.size
+    feed.items.each do |item|
+      if SITE_KEYS.include?(item.author)
+        author = item.author.strip
+        category = item.category.to_s.gsub(/\<(\/)*category\>/, '')
+        if all[author]
+          all[author][item.link.strip] = {title: item.title.strip, author: author, link: item.link.strip, date: item.pubDate, category: category, description: item.description.gsub("\r\n", '<br/>').strip}
+        else
+          all[author] = {item.link.strip => {title: item.title.strip, author: author, link: item.link.strip, date: item.pubDate, category: category, description: item.description.gsub("\r\n", '<br/>').strip}}
+        end
+      end
+    end
+    continue = false if feed.items.size < 100
+  end
+  i += 1
+end
+
+# Get corresponding category_id or create a new one
+def get_category_id(category, categories, coll_cat)
+  if categories.keys.include? "rss_#{category}"
+    [categories["rss_#{category}"], categories]
+  else
+    cat = {
+      _type: "BulletinCategory",
+      key: "rss_#{category}",
+      disable: false,
+      title: {:zh_tw => category},
+      created_at: Time.now,
+      updated_at: Time.now
+    }
+    categories["rss_#{category}"] = result = coll_cat.save(cat)
+    [result, categories]
+  end
+end
+
+# Get categories and id based on a given site number
+def get_mongo_and_categories(site_number="0")
+  db = Mongo::Connection.new("localhost", 27017).db("#{DB_BASE_NAME}_#{site_number}")
+  coll_bulletin = db["bulletins"]
+  coll_cat =  db["bulletin_categories"]
+
+  categories = coll_cat.find().to_a.inject({}) do |categories, category|
+    categories[category['key']] = category['_id']
+    categories
+  end
+  [categories, coll_bulletin, coll_cat]
+end
+
+# Get main site (總務處) categories
+@main_categories, @main_coll_bulletin, @main_coll_cat = get_mongo_and_categories
+
+all.each do |key, value| # Loop through all the authors
+  site_number = SITES[key]
+  categories, coll_bulletin, coll_cat = get_mongo_and_categories(site_number) # Get current's site categories
+  value.each_value do |bul| # Loop through all the items
+    category_id, categories = get_category_id(bul[:category], categories, coll_cat)
+    unless coll_bulletin.find_one(rss_link: bul[:link]) 
+      bulletin = {  _type: "Bulletin",
+                    postdate: bul[:date],
+                    created_at: bul[:date],
+                    updated_at: bul[:date],
+                    is_checked: true,
+                    is_pending: false,
+                    is_rejected: false,
+                    bulletin_category_id: category_id,
+                    title: {:zh_tw => bul[:title]},
+                    text: {:zh_tw => bul[:description]},
+                    available_for_zh_tw: true,
+                    rss_link: bul[:link],
+                    is_top: false,
+                    is_hot: false,
+                    is_hidden: false }
+      coll_bulletin.save(bulletin) 
+      unless site_number.eql? "0" # Copy the item to the main site
+        category_id, @main_categories = get_category_id(bul[:category], @main_categories, @main_coll_cat)
+        main_bulletin = bulletin.clone
+        main_bulletin['_id'] = BSON::ObjectId.new
+        main_bulletin[:bulletin_category_id] = category_id
+        @main_coll_bulletin.save(main_bulletin)
+      end
+    end
+  end
+end
+
+

lib/tasks/migrate.rake

@@ -320,6 +320,46 @@ namespace :migrate do
         bull.update_attribute(:available_for_zh_tw, true)
       end
     end
+  end
+
+  task :fix_imported_data => :environment do
+    categories = BulletinCategory.where(key: /import_/).entries
+    unless categories.blank?
+      categories.each do |category|
+        bulletins = category.bulletins
+        unless bulletins.blank?
+          bulletins.each do |bulletin|
+            unless bulletin.bulletin_files.blank?
+              file = bulletin.bulletin_files[0]
+              bulletin.text_translations = file.description_translations
+              bulletin.save
+              file.update_attribute(:description, nil)
+            end
+          end
+        end
+      end
+    end
+    categories = ArchiveFileCategory.where(key: /import_/).entries
+    unless categories.blank?
+      categories.each do |category|
+        files = category.archive_files
+        unless files.blank?
+          files.each{|file| file.update_attributes({is_top: false, is_hot: false, is_hidden: false})}
+        end
+      end
+    end
+  end 
+
+  task :delete_rss_announcement => :environment do
+    categories = BulletinCategory.where(key: /rss_/).entries
+    unless categories.blank?
+      categories.each do |category|
+        bulletins = category.bulletins
+        unless bulletins.blank?
+          bulletins.each(&:destroy)
+        end
+      end
+    end
   end 
 
   task :fix_imported_rss => :environment do