Add manual RSS/JSON subscription with configurable field mapping

app/assets/javascripts/new_channel.js

@@ -12,6 +12,206 @@
       $("#site-url-wrapper").fadeOut();
       getChannelList();
     })
+    $("#toggle-manual-rss").on("click",function(){
+      $("#manual-rss-form").slideToggle();
+    })
+    $("input[name='source-type']").on("change",function(){
+      if($(this).val() == "json"){
+        $("#rss-detect-block").hide();
+        $("#json-detect-block").show();
+      }else{
+        $("#json-detect-block").hide();
+        $("#rss-detect-block").show();
+      }
+    })
+
+    $("#manual-rss-detect").on("click",function(){
+      var rss_url = $("#manual-rss-url").val().trim();
+      if(!rss_url){
+        alert("請先填寫 RSS 網址");
+        return;
+      }
+      $.ajax({
+        url: "/admin/feeds/rss_fields",
+        data: {"url": rss_url},
+        dataType: "json",
+        type: "get"
+      }).done(function(data){
+        if(!data.fields || data.fields.length == 0){
+          alert("偵測不到欄位，請確認 RSS 網址是否正確");
+          return;
+        }
+        var titleSelect = $("#manual-rss-title-field");
+        var linkSelect = $("#manual-rss-link-field");
+        var imageSelect = $("#manual-rss-image-field");
+        titleSelect.html("");
+        linkSelect.html("");
+        imageSelect.html("<option value=''>（不設定圖片欄位）</option>");
+        $.each(data.fields, function(i, f){
+          titleSelect.append("<option value='" + f + "'" + (f=="title" ? " selected" : "") + ">" + f + "</option>");
+          linkSelect.append("<option value='" + f + "'" + (f=="link" ? " selected" : "") + ">" + f + "</option>");
+          imageSelect.append("<option value='" + f + "'" + (f=="imageUrl" ? " selected" : "") + ">" + f + "</option>");
+        })
+        $("#manual-rss-mapping").show();
+      })
+    })
+
+    $("#json-detect-step1").on("click",function(){
+      var json_url = $("#manual-rss-url").val().trim();
+      if(!json_url){
+        alert("請先填寫網址");
+        return;
+      }
+      $.ajax({
+        url: "/admin/feeds/json_top_keys",
+        data: {"url": json_url},
+        dataType: "json",
+        type: "get"
+      }).done(function(data){
+        if(data.error){
+          alert("抓取失敗，請確認網址是否正確、是否為 JSON 格式");
+          return;
+        }
+        var select = $("#json-list-key-select");
+        select.html("");
+        if(data.root_is_array){
+          select.append("<option value=''>（根層級本身就是陣列）</option>");
+        }
+        $.each(data.array_keys, function(i, k){
+          select.append("<option value='" + k + "'>" + k + "</option>");
+        })
+        if(!data.root_is_array && data.array_keys.length == 0){
+          alert("偵測不到任何陣列欄位，請確認這個網址回傳的是不是文章列表 JSON");
+          return;
+        }
+        $("#json-list-key-wrapper").show();
+        $("#json-field-mapping").hide();
+      })
+    })
+
+    $("#json-detect-step2").on("click",function(){
+      var json_url = $("#manual-rss-url").val().trim();
+      var list_key = $("#json-list-key-select").val();
+      $.ajax({
+        url: "/admin/feeds/json_item_fields",
+        data: {"url": json_url, "list_key": list_key},
+        dataType: "json",
+        type: "get"
+      }).done(function(data){
+        if(!data.fields || data.fields.length == 0){
+          alert("偵測不到欄位，請確認列表位置選對了嗎");
+          return;
+        }
+        var titleSelect = $("#json-title-field");
+        var subtitleSelect = $("#json-subtitle-field");
+        var contentSelect = $("#json-content-field");
+        var timeSelect = $("#json-time-field");
+        var imageSelect = $("#json-image-field");
+        var idSelect = $("#json-id-field");
+        var linkFieldSelect = $("#json-link-field");
+        titleSelect.html("");
+        subtitleSelect.html("<option value=''>（不設定副標題欄位）</option>");
+        contentSelect.html("<option value=''>（不設定內容欄位）</option>");
+        timeSelect.html("");
+        imageSelect.html("<option value=''>（不設定圖片欄位）</option>");
+        idSelect.html("");
+        linkFieldSelect.html("");
+        $.each(data.fields, function(i, f){
+          titleSelect.append("<option value='" + f + "'>" + f + "</option>");
+          subtitleSelect.append("<option value='" + f + "'>" + f + "</option>");
+          contentSelect.append("<option value='" + f + "'>" + f + "</option>");
+          timeSelect.append("<option value='" + f + "'>" + f + "</option>");
+          imageSelect.append("<option value='" + f + "'>" + f + "</option>");
+          idSelect.append("<option value='" + f + "'>" + f + "</option>");
+          linkFieldSelect.append("<option value='" + f + "'" + ((f=="url"||f=="link") ? " selected" : "") + ">" + f + "</option>");
+        })
+        $("#json-field-mapping").show();
+      })
+    })
+    
+    $("input[name='json-link-mode']").on("change",function(){
+      if($(this).val() == "template"){
+        $("#json-link-field-wrapper").hide();
+        $("#json-link-template-wrapper").show();
+      }else{
+        $("#json-link-template-wrapper").hide();
+        $("#json-link-field-wrapper").show();
+      }
+    })
+
+    $("#manual-rss-next").on("click",function(){
+      var rss_url = $("#manual-rss-url").val().trim();
+      var title = $("#manual-rss-title").val().trim();
+      var source_type = $("input[name='source-type']:checked").val();
+      var channel_key = "announcement";
+
+      if(!rss_url || !title){
+        alert("請把網址、頻道顯示名稱填寫完整");
+        return;
+      }
+
+      site_url = rss_url;
+      var title_translations = {};
+      title_translations[I18n.locale] = title;
+      var feed = {
+        uid: title.replace(/\s+/g,'_') + "_" + Date.now(),
+        url: rss_url,
+        title_translations: title_translations
+      };
+
+      if(source_type == "json"){
+        var list_key = $("#json-list-key-select").val();
+        var json_title_field = $("#json-title-field").val();
+        var json_subtitle_field = $("#json-subtitle-field").val();
+        var json_content_field = $("#json-content-field").val();
+        var json_time_field = $("#json-time-field").val();
+        var json_image_field = $("#json-image-field").val();
+        var json_id_field = $("#json-id-field").val();
+        var json_link_mode = $("input[name='json-link-mode']:checked").val();
+        var json_link_field = $("#json-link-field").val();
+        var json_link_template = $("#json-link-template").val().trim();
+
+        if(!json_title_field || !json_id_field){
+          alert("請完成 JSON 欄位偵測，並至少填寫標題欄位、ID欄位");
+          return;
+        }
+        if(json_link_mode == "template" && !json_link_template){
+          alert("請填寫連結樣板");
+          return;
+        }
+        if(json_link_mode == "field" && !json_link_field){
+          alert("請選擇連結欄位");
+          return;
+        }
+
+        feed.is_json_list = true;
+        feed.json_list_key = list_key;
+        feed.json_id_field = json_id_field;
+        feed.json_title_field = json_title_field;
+        feed.json_subtitle_field = json_subtitle_field;
+        feed.json_content_field = json_content_field;
+        feed.json_time_field = json_time_field;
+        feed.json_image_field = json_image_field;
+        feed.json_link_mode = json_link_mode;
+        feed.json_link_field = json_link_field;
+        feed.json_link_template = json_link_template;
+      }else{
+        var title_field = $("#manual-rss-title-field").val();
+        var link_field = $("#manual-rss-link-field").val();
+        var image_field = $("#manual-rss-image-field").val();
+        if(!title_field || !link_field){
+          alert("請把標題欄位、連結欄位填寫完整");
+          return;
+        }
+        feed.is_rss = true;
+        feed.rss_title_field = title_field;
+        feed.rss_link_field = link_field;
+        feed.rss_image_field = image_field;
+      }
+
+      var channel = {key: channel_key, title: title};
+      subscribeFeed($("<div></div>"), feed, channel);
+    })
   }
 
   var getChannelList = function(){

app/assets/stylesheets/new_channel.css

@@ -106,3 +106,15 @@
     margin-right: 10px; 
     vertical-align: top;
   }
+    #manual-rss-wrapper{
+    width: 700px;
+    margin: 10px auto 0;
+    text-align: center;
+  }
+  #manual-rss-form{
+    margin-top: 8px;
+  }
+  .manual-rss-input{
+    width: 680px;
+    margin-bottom: 8px;
+  }

app/controllers/admin/feeds_controller.rb

@@ -162,6 +162,54 @@ class Admin::FeedsController < OrbitAdminController
 		render :json => data.to_json
 	end
         
+        def rss_fields
+		fields = []
+		begin
+			uri = URI(params[:url])
+			body = http_request(uri).body
+			rss_data = Hash.from_xml(body)
+			items = rss_data['rss']['channel']['item']
+			first_item = items.is_a?(Array) ? items.first : items
+			fields = first_item.keys if first_item.is_a?(Hash)
+		rescue => e
+			fields = []
+		end
+		render :json => {"fields" => fields}.to_json
+	end
+
+        def json_top_keys
+		data = {}
+		begin
+			uri = URI(params[:url])
+			body = http_request(uri).body
+			json_data = JSON.parse(body)
+			if json_data.is_a?(Array)
+				data = {"root_is_array" => true, "array_keys" => []}
+			else
+				array_keys = json_data.select{|k,v| v.is_a?(Array)}.keys
+				data = {"root_is_array" => false, "array_keys" => array_keys}
+			end
+		rescue => e
+			data = {"error" => true}
+		end
+		render :json => data.to_json
+	end
+
+	def json_item_fields
+		fields = []
+		begin
+			uri = URI(params[:url])
+			body = http_request(uri).body
+			json_data = JSON.parse(body)
+			items = SiteFeed.dig_by_path(json_data, params[:list_key])
+			first_item = Array(items).first
+			fields = first_item.keys if first_item.is_a?(Hash)
+		rescue => e
+			fields = []
+		end
+		render :json => {"fields" => fields}.to_json
+	end
+
 	def get_feed_list
 		data_to_send = {}
 		if params['feeds']
@@ -217,10 +265,32 @@ class Admin::FeedsController < OrbitAdminController
 		site_feed.feed_name_translations = params[:feed][:title_translations]
 		site_feed.disabled = false
 		site_feed.feed_url = params[:feed][:url]
-		if params[:feed][:is_rss]
+                if params[:feed][:is_rss]
 			site_feed.is_rss = params[:feed][:is_rss]
 		end
-		uri = URI(site_feed.remote_site_url)
+                if params[:feed][:rss_title_field].present?
+			site_feed.rss_title_field = params[:feed][:rss_title_field]
+		end
+		if params[:feed][:rss_link_field].present?
+			site_feed.rss_link_field = params[:feed][:rss_link_field]
+		end
+		if params[:feed][:rss_image_field].present?
+			site_feed.rss_image_field = params[:feed][:rss_image_field]
+		end
+		if params[:feed][:is_json_list]
+			site_feed.is_json_list = params[:feed][:is_json_list]
+			site_feed.json_list_key = params[:feed][:json_list_key]
+			site_feed.json_id_field = params[:feed][:json_id_field]
+			site_feed.json_title_field = params[:feed][:json_title_field]
+			site_feed.json_subtitle_field = params[:feed][:json_subtitle_field]
+			site_feed.json_content_field = params[:feed][:json_content_field]
+			site_feed.json_time_field = params[:feed][:json_time_field]
+			site_feed.json_image_field = params[:feed][:json_image_field]
+			site_feed.json_link_mode = params[:feed][:json_link_mode]
+			site_feed.json_link_field = params[:feed][:json_link_field]
+			site_feed.json_link_template = params[:feed][:json_link_template]
+		end
+                uri = URI(site_feed.remote_site_url)
 		site_host = uri.host
 		other_site_feeds = SiteFeed.where(:remote_site_url=>/:\/\/#{::Regexp.escape(site_host)}/)
 		other_site_feed = other_site_feeds.first

app/models/site_feed.rb

@@ -20,6 +20,20 @@ class SiteFeed
   field :feeds_update_statuses_callback
   field :enable_notify, type: Boolean, default: false
   field :is_rss, type: Boolean, default: false
+  field :rss_title_field, default: "title"
+  field :rss_link_field, default: "link"
+  field :rss_image_field
+  field :is_json_list, type: Boolean, default: false
+  field :json_list_key         
+  field :json_id_field
+  field :json_title_field
+  field :json_subtitle_field
+  field :json_content_field
+  field :json_time_field
+  field :json_image_field
+  field :json_link_mode, default: "field"   
+  field :json_link_field
+  field :json_link_template
   require 'fileutils'
   include Feeds::Cache
   Category.send(:include,Feeds::Cache)
@@ -103,15 +117,20 @@ class SiteFeed
         rss_data = Hash.from_xml(res)
         category_title = rss_data['rss']['channel']['title']
         category_title_trans = to_I18n(category_title, true)
+        title_field = self.rss_title_field.presence || "title"
+        link_field = self.rss_link_field.presence || "link"
+        image_field = self.rss_image_field
         res_to_json = rss_data['rss']['channel']["item"].collect do |item|
           tag_titles = Array(item['類別']).collect{|title| to_I18n(title, true, "name_translations")}
-          id = item['link'].split("&s=")[-1]
+          item_link = item[link_field]
-          title = Array(item['title'])[0]
+          id = item_link.split("&s=")[-1]
+          title = Array(item[title_field])[0]
           text = Array(item['description'])[0]
+          image_url = image_field.present? ? item[image_field] : nil
           files = []
           if text.include?('附件')
-            files = Nokogiri::HTML(get_response_body(URI(item['link']))).search('#RelData1_liFile ol a').collect do |v|
+            files = Nokogiri::HTML(get_response_body(URI(item_link))).search('#RelData1_liFile ol a').collect do |v|  
-              file_title = v.attr('title')
+            file_title = v.attr('title')
               {
                 "description_translations" => to_I18n(file_title),
                 "title_translations" => to_I18n(file_title.sub(/^\[(?:(?!\]).)*\]/,'').rpartition(".")[0]),
@@ -128,7 +147,11 @@ class SiteFeed
             "text_translations" => to_I18n(text),
             "postdate" => DateTime.parse(item['pubDate']).to_s,
             "image_description_translations" => {},
-            "image" => {},
+            "image" => image_url.blank? ? {} : {
+              "thumb" => image_url,
+              "mobile" => image_url,
+              "original" => image_url
+            },
             "display_img" => false,
             "tags" => tag_titles,
             "category" => category_title_trans,
@@ -137,7 +160,57 @@ class SiteFeed
             "bulletin_links" => [],
             "bulletin_files" => files,
             "bulletin_carousel_images" => [],
-            "external_link" => nil
+            "external_link" => item_link
+          }
+        end
+        res = JSON.dump({"announcements" => res_to_json})
+      elsif self.is_json_list
+        json_data = JSON.parse(res) rescue {}
+        items = self.class.dig_by_path(json_data, self.json_list_key)
+        id_field = self.json_id_field
+        title_field = self.json_title_field
+        subtitle_field = self.json_subtitle_field
+        content_field = self.json_content_field
+        time_field = self.json_time_field
+        image_field = self.json_image_field
+        link_mode = self.json_link_mode.presence || "field"
+        link_field = self.json_link_field
+        link_template = self.json_link_template
+        res_to_json = Array(items).collect do |item|
+          id = id_field.present? ? item[id_field].to_s : nil
+          if link_mode == "template"
+            link = (link_template.present? && id.present?) ? link_template.gsub("{id}", id) : nil
+          else
+            link = link_field.present? ? item[link_field] : nil
+          end
+          title = title_field.present? ? item[title_field] : nil  
+          subtitle = subtitle_field.present? ? item[subtitle_field] : nil
+          content = content_field.present? ? item[content_field] : nil
+          postdate = (time_field.present? ? (DateTime.parse(item[time_field]) rescue DateTime.now) : DateTime.now).to_s
+          image_url = image_field.present? ? item[image_field] : nil
+          {
+            "show_url" => link,
+            "org_is_top" => 0,
+            "id" => id,
+            "title_translations" => to_I18n(title),
+            "subtitle_translations" => to_I18n(subtitle),
+            "text_translations" => to_I18n(content),
+            "postdate" => postdate,
+            "image_description_translations" => {},
+            "image" => image_url.blank? ? {} : {
+              "thumb" => image_url,
+              "mobile" => image_url,
+              "original" => image_url
+            },
+            "display_img" => !image_url.blank?,
+            "tags" => [],
+            "category" => category_title_trans,
+            "author" => nil,
+            "params" => "#{title}-#{id}",
+            "bulletin_links" => [],
+            "bulletin_files" => [],
+            "bulletin_carousel_images" => [],
+            "external_link" => link
           }
         end
         res = JSON.dump({"announcements" => res_to_json})
@@ -151,6 +224,12 @@ class SiteFeed
     end
     anns
   end
+
+  def self.dig_by_path(data, path)
+    return data if path.blank?
+    path.split(".").inject(data) { |acc, key| acc.is_a?(Hash) ? acc[key] : nil }
+  end
+
   def get_response_body(uri)
     res = Net::HTTP.start(uri.host, uri.port,
                           :use_ssl => uri.scheme == 'https',

app/views/admin/feeds/new.html.erb

@@ -7,7 +7,7 @@
 <div id="import-container">
   <div id="import-head" class="muted">
     <i class='icons-rss'></i><br/>
-    Subscribe to channels
+    Subscribe to Orbit channels
   </div>
 
   <div id="loading" style="display:none;">
@@ -16,10 +16,73 @@
   </div>
 
   <div id="site-url-wrapper" class="import-wrapper">
-    <input type="text" class="import-url" name="url" value="" placeholder="http://www.rulingcom.com">
+    <input type="text" class="import-url" name="url" value="" placeholder="http://orbitek.co">
     <button id="find-channels" class="btn btn-primary">Find Channels</button>
   </div>
 
+  <!-- === 新增區塊：手動輸入 RSS 網址訂閱公告 === -->
+  <div id="manual-rss-wrapper" class="import-wrapper">
+    <a href="javascript:void(0);" id="toggle-manual-rss">或者，直接輸入網址訂閱公告</a>
+    <div id="manual-rss-form" style="display:none;">
+
+      <div id="manual-source-type">
+        <label><input type="radio" name="source-type" value="rss" checked> RSS 來源</label>
+        &nbsp;&nbsp;
+        <label><input type="radio" name="source-type" value="json"> JSON API 來源</label>
+      </div>
+      <input type="text" id="manual-rss-url" class="manual-rss-input" placeholder="網址，例如 https://www.peopo.org/rss-top10"><br/>
+
+      <!-- RSS 偵測區塊 -->
+      <div id="rss-detect-block">
+        <button id="manual-rss-detect" class="btn">偵測欄位</button><br/>
+        <div id="manual-rss-mapping" style="display:none;">
+          標題欄位：<select id="manual-rss-title-field" class="manual-rss-input"></select><br/>
+          連結欄位：<select id="manual-rss-link-field" class="manual-rss-input"></select><br/>
+          圖片欄位：<select id="manual-rss-image-field" class="manual-rss-input">
+            <option value="">（不設定圖片欄位）</option>
+          </select><br/>
+        </div>
+      </div>
+
+      <!-- JSON 偵測區塊（新增） -->
+      <div id="json-detect-block" style="display:none;">
+        <button id="json-detect-step1" class="btn">第一步：抓取結構</button><br/>
+        <div id="json-list-key-wrapper" style="display:none;">
+          文章列表位置：<select id="json-list-key-select" class="manual-rss-input"></select>
+          <button id="json-detect-step2" class="btn">第二步：抓取欄位</button><br/>
+        </div>
+        <div id="json-field-mapping" style="display:none;">
+          標題欄位：<select id="json-title-field" class="manual-rss-input"></select><br/>
+          副標題欄位：<select id="json-subtitle-field" class="manual-rss-input">
+            <option value="">（不設定副標題欄位）</option>
+          </select><br/>
+          內容欄位：<select id="json-content-field" class="manual-rss-input">
+            <option value="">（不設定內容欄位）</option>
+          </select><br/>
+          時間欄位：<select id="json-time-field" class="manual-rss-input"></select><br/>
+          圖片欄位：<select id="json-image-field" class="manual-rss-input">
+            <option value="">（不設定圖片欄位）</option>
+          </select><br/>
+          ID欄位：<select id="json-id-field" class="manual-rss-input"></select><br/>
+
+          連結來源：
+          <label><input type="radio" name="json-link-mode" value="field" checked> 直接讀取欄位</label>
+          <label><input type="radio" name="json-link-mode" value="template"> 用樣板組出來</label><br/>
+
+          <div id="json-link-field-wrapper">
+            連結欄位：<select id="json-link-field" class="manual-rss-input"></select><br/>
+          </div>
+          <div id="json-link-template-wrapper" style="display:none;">
+            連結樣板：<input type="text" id="json-link-template" class="manual-rss-input" placeholder="例如 https://news.pts.org.tw/article/{id}"><br/>
+          </div>
+        </div>
+      </div>
+
+      <input type="text" id="manual-rss-title" class="manual-rss-input" placeholder="頻道顯示名稱，例如 PeoPo公民新聞"><br/>
+      <button id="manual-rss-next" class="btn btn-primary">下一步：選擇分類</button>
+    </div>
+  </div>
+
   <div id="channels" style="display:none; text-align:center;">
   </div>
 </div>

config/routes.rb

@@ -4,6 +4,9 @@ Rails.application.routes.draw do
   scope "(:locale)", locale: Regexp.new(locales.join("|")) do
    namespace :admin do
     get "/feeds/get_channel_list", to: 'feeds#get_channel_list'
+    get "/feeds/rss_fields", to: 'feeds#rss_fields'
+    get "/feeds/json_top_keys", to: 'feeds#json_top_keys'
+    get "/feeds/json_item_fields", to: 'feeds#json_item_fields'
     post "/feeds/get_feed_list", to: 'feeds#get_feed_list'
     get "/feeds/get_category_list", to: 'feeds#get_category_list'
     post "/feeds/subscribe", to: 'feeds#subscribe'