☰

HIT Scraper (classic version)

Snag HITs.

  1. // ==UserScript==
  2. // @name        HIT Scraper (classic version)
  3. // @author      Kerek
  4. // @description Snag HITs.
  5. //              Based in part on code from mmmturkeybacon Export Mturk History and mmmturkeybacon Color Coded Search with Checkpoints
  6. // @namespace   http://userscripts.org/users/536998
  7. // @match       https://www.mturk.com/mturk/findhits?match=true#hit_scraper*
  8. // @match       https://www.mturk.com/mturk/findhits?match=true?hit_scraper*
  9. // @version     1.3.0.1
  10. // @grant       GM_xmlhttpRequest
  11. // @grant       GM_getValue
  12. // @grant       GM_setValue
  13. // @require     http://code.jquery.com/jquery-latest.min.js
  14. // ==/UserScript==
  15.  
  16. //alter the requester ignore last as you desire, just follow the format below and use EXACT capitalization e.g., CrowdSource
  17. var ignore_list = ["Oscar Smith", "Jon Brelig"];
  18.  
  19. //this searches extra pages if you skip too much, helps fill out results if you hit a chunk of ignored HITs.  Change to true for this behavior.
  20. var correct_for_skips = false;
  21.  
  22. //weight the four TO ratings for the coloring. Default has pay twice as important as fairness and nothing for communication and fast.
  23. var COMM_WEIGHT = 0;
  24. var PAY_WEIGHT  = 10;
  25. var FAIR_WEIGHT = 5;
  26. var FAST_WEIGHT = 0;
  27.  
  28. //display your hitdb records if applicable
  29. var check_hitDB = true;
  30.  
  31. //default text size
  32. var default_text_size=11;
  33.  
  34.  
  35.  
  36. var HITStorage = {};
  37. var indexedDB = window.indexedDB || window.webkitIndexedDB ||
  38.                 window.mozIndexedDB;
  39. window.IDBTransaction = window.IDBTransaction || window.webkitIDBTransaction || window.mozIDBTransaction;
  40. window.IDBKeyRange = window.IDBKeyRange || window.webkitIDBKeyRange || window.mozIDBKeyRange;
  41. HITStorage.IDBTransactionModes = { "READ_ONLY": "readonly", "READ_WRITE": "readwrite", "VERSION_CHANGE": "versionchange" };
  42. var IDBKeyRange = window.IDBKeyRange;
  43.  
  44. HITStorage.indexedDB = {};
  45. HITStorage.indexedDB = {};
  46. HITStorage.indexedDB.db = null;
  47.  
  48. HITStorage.indexedDB.onerror = function(e) {
  49.   console.log(e);
  50. };
  51.  
  52. var v=4;
  53.  
  54. HITStorage.indexedDB.checkTitle = function(title,button) {
  55.   var request = indexedDB.open("HITDB", v);
  56. 	request.onsuccess = function(e) {
  57. 		HITStorage.indexedDB.db = e.target.result;
  58. 		var db = HITStorage.indexedDB.db;
  59.     if (!db.objectStoreNames.contains("HIT"))
  60.     {
  61.       db.close();
  62.       return;
  63.     }
  64. 		var trans = db.transaction(["HIT"], HITStorage.IDBTransactionModes.READ_ONLY);
  65. 		var store = trans.objectStore("HIT");
  66.  
  67. 		var index = store.index("title");
  68.     index.get(title).onsuccess = function(event)
  69.     {
  70.       if (event.target.result === undefined)
  71.       {
  72.              console.log(title + ' not found');
  73.         history[button].titledb=false;
  74.       }
  75.       else
  76.       {
  77.           console.log(title + ' found');
  78.         history[button].titledb=true;
  79.       }
  80.       
  81.       db.close();
  82.     };
  83. 	};
  84. 	request.onerror = HITStorage.indexedDB.onerror;
  85. };
  86.  
  87. HITStorage.indexedDB.checkRequester = function(id,button) {
  88.   var request = indexedDB.open("HITDB", v);
  89. 	request.onsuccess = function(e) {
  90. 		HITStorage.indexedDB.db = e.target.result;
  91. 		var db = HITStorage.indexedDB.db;
  92.     if (!db.objectStoreNames.contains("HIT"))
  93.     {
  94.       db.close();
  95.       return;
  96.     }
  97. 		var trans = db.transaction(["HIT"], HITStorage.IDBTransactionModes.READ_ONLY);
  98. 		var store = trans.objectStore("HIT");
  99.  
  100. 		var index = store.index("requesterId");
  101.     index.get(id).onsuccess = function(event)
  102.     {
  103.       if (event.target.result === undefined)
  104.       {history[button].reqdb=false;
  105.         console.log(id + ' not found');
  106.       }
  107.       else
  108.       {
  109.             history[button].reqdb=true;
  110.         console.log(id + ' found');
  111.       }
  112.       db.close();
  113.     };
  114. 	};
  115. 	request.onerror = HITStorage.indexedDB.onerror;
  116. };
  117.  
  118. var PAGES_TO_SCRAPE = 3;
  119. var MINIMUM_HITS = 100;
  120. var SEARCH_REFRESH=0;
  121. var URL_BASE = "/mturk/searchbar?searchWords=&selectedSearchType=hitgroups";
  122. var initial_url = URL_BASE;
  123. var TO_REQ_URL = "http://turkopticon.ucsd.edu/reports?id=";
  124. var found_key_list=[];
  125. var last_clear_time = new Date().getTime();
  126. var searched_once = false;
  127. var save_new_results_time = 120;
  128. var save_results_time = 3600;
  129. var default_type = 0;
  130. var cur_loc = window.location.href;
  131. var time_input = document.createElement("INPUT");
  132. time_input.value = 0;
  133. var page_input = document.createElement("INPUT");
  134. page_input.value = 3;
  135. var min_input = document.createElement("INPUT");
  136. var new_time_display_input = document.createElement("INPUT");
  137. new_time_display_input.value = 300;
  138. var reward_input = document.createElement("INPUT");
  139. var qual_input = document.createElement("INPUT");
  140. qual_input.type = "checkbox";
  141. qual_input.checked = true;
  142. var masters_input = document.createElement("INPUT");
  143. masters_input.type = "checkbox";
  144. var sort_input1 = document.createElement("INPUT");
  145. sort_input1.type = "radio";
  146. sort_input1.name = "sort_type";
  147. sort_input1.value = "latest";
  148. sort_input1.checked = true;
  149. var sort_input2 = document.createElement("INPUT");
  150. sort_input2.type = "radio";
  151. sort_input2.name = "sort_type";
  152. sort_input2.value = "most";
  153. var sort_input3 = document.createElement("INPUT");
  154. sort_input3.type = "radio";
  155. sort_input3.name = "sort_type";
  156. sort_input3.value = "amount";
  157.  
  158. var search_input = document.createElement("INPUT");
  159.  
  160. var LINK_BASE = "https://www.mturk.com";
  161. var BACKGROUND_COLOR = "rgb(19, 19, 19)";
  162. var STATUSDETAIL_DELAY = 250;
  163. var MPRE_DELAY = 3000;
  164.  
  165. var next_page = 1;
  166.  
  167. var GREEN   = '#66CC66'; //  > 4
  168. var LIGHTGREEN = '#ADFF2F'; // > 3  GREEN YELLOW
  169. var YELLOW = '#FFD700';
  170. var ORANGE  = '#FF9900'; //  > 2
  171. var RED     = '#FF3030'; // <= 2
  172. var BLUE    = '#C0D9D9'; // no TO
  173. var GREY = 'lightGrey';
  174. var BROWN = '#94704D';
  175. var DARKGREY = '#9F9F9F';
  176. $('body').css('background', BACKGROUND_COLOR);
  177.  
  178. var API_PROXY_BASE = 'https://mturk-api.istrack.in/';
  179. var API_MULTI_ATTRS_URL = API_PROXY_BASE + 'multi-attrs.php?ids=';
  180. var REVIEWS_BASE = 'http://turkopticon.ucsd.edu/';
  181.  
  182. var control_panel_HTML = '<div id="control_panel" style="margin: 0 auto 0 auto;' +
  183.                          'border-bottom: 1px solid #000000; margin-bottom: 5px; ' +
  184.                          'background-color: ' + BACKGROUND_COLOR + ';"></div>';
  185. $('body > :not(#control_panel)').hide(); //hide all nodes directly under the body
  186. $('body').prepend(control_panel_HTML);
  187.  
  188. var control_panel = document.getElementById("control_panel");
  189. var big_red_button = document.createElement("BUTTON");
  190. var progress_report = document.createTextNode("Stopped");
  191. var text_area = document.createElement("TABLE");
  192. big_red_button.textContent = "Show Interface";
  193. big_red_button.onclick =  function(){show_interface();};
  194. control_panel.appendChild(big_red_button);
  195.  
  196. show_interface();
  197.  
  198. var global_run = false;
  199. var statusdetail_loop_finished = false;
  200. var date_header = "";
  201. var history = {};
  202. var wait_loop;
  203.  
  204. function set_progress_report(text, force)
  205. {
  206.     if (global_run == true || force == true)
  207.     {
  208.         progress_report.textContent = text;
  209.     }
  210. }
  211.  
  212. function get_progress_report()
  213. {
  214.     return progress_report.textContent;
  215. }
  216.  
  217. function wait_until_stopped()
  218. {
  219.     if (global_run == true)
  220.     {
  221.         if (statusdetail_loop_finished == true)
  222.         {
  223.             big_red_button.textContent = "Start";
  224.             set_progress_report("Finished", false);
  225.         }
  226.         else
  227.         {
  228.             setTimeout(function(){wait_until_stopped();}, 500);
  229.         }
  230.     }
  231. }
  232.  
  233. function display_wait_time(wait_time)
  234. {
  235.     if (global_run == true)
  236.     {
  237.         var current_progress = get_progress_report();
  238.         if (current_progress.indexOf("Searching again in")!==-1)
  239.         {
  240.             set_progress_report(current_progress.replace(/Searching again in \d+ seconds/ , "Searching again in " + wait_time + " seconds"),false);
  241.         }
  242.         else
  243.             set_progress_report(current_progress + " Searching again in " + wait_time + " seconds.", false);
  244.         if (wait_time>1)
  245.             setTimeout(function(){display_wait_time(wait_time-1);}, 1000);
  246.     }
  247. }
  248.  
  249. function dispArr(ar)
  250. {
  251.     var disp = "";
  252.     for (var z = 0; z < ar.length; z++)
  253.     {
  254.         disp += "id " + z + " is " + ar[z] + " ";
  255.     }
  256.     console.log(disp);
  257. }
  258.  
  259. function scrape($src)
  260. {
  261.     var $requester = $src.find('a[href^="/mturk/searchbar?selectedSearchType=hitgroups&requester"]');
  262.     var $title = $src.find('a[class="capsulelink"]');
  263.     var $reward = $src.find('span[class="reward"]');
  264.     var $preview = $src.find('a[href^="/mturk/preview?"]');
  265.     var $qualified = $src.find('a[href^="/mturk/notqualified?"]');
  266.     var not_qualified_group_IDs=[];
  267.     $qualified.each(function(){
  268.         var groupy = $(this).attr('href');
  269.         groupy = groupy.replace("/mturk/notqualified?hitId=","");
  270.         not_qualified_group_IDs.push(groupy);
  271.     });
  272.     var $mixed =  $src.find('a[href^="/mturk/preview?"],a[href^="/mturk/notqualified?"]');
  273.     var listy =[];
  274.     $mixed.each(function(){
  275.         var groupy = $(this).attr('href');
  276.         groupy = groupy.replace("/mturk/notqualified?hitId=","");
  277.         groupy = groupy.replace("/mturk/preview?groupId=","");
  278.         listy.push(groupy);
  279.     });
  280.     listy = listy.filter(function(elem, pos) {
  281.         return listy.indexOf(elem) == pos;
  282.     });
  283.  
  284.     for (var j = 0; j < $requester.length; j++)
  285.     {
  286.         var $hits = $requester.eq(j).parent().parent().parent().parent().parent().parent().find('td[class="capsule_field_text"]');
  287.         var requester_name = $requester.eq(j).text().trim();
  288.         var requester_link = $requester.eq(j).attr('href');
  289.         var group_ID=listy[j];
  290.         var preview_link = "/mturk/preview?groupId=" + group_ID;
  291.         var title = $title.eq(j).text().trim();
  292.         var reward = $reward.eq(j).text().trim();
  293.         var hits = $hits.eq(4).text().trim();
  294.         var requester_id = requester_link.replace('/mturk/searchbar?selectedSearchType=hitgroups&requesterId=','');
  295.         var accept_link;
  296.         accept_link = preview_link.replace('preview','previewandaccept');
  297.  
  298.         key = requester_name+title+reward+group_ID;
  299.         found_key_list.push(key);
  300.         if (history[key] == undefined)
  301.         {
  302.             history[key] = {requester:"", title:"", reward:"", hits:"", req_link:"", prev_link:"", rid:"", acc_link:"", new_result:"", qualified:"", found_this_time:"", initial_time:"", reqdb:"",titledb:""};
  303.             history[key].req_link = requester_link;
  304.             history[key].prev_link = preview_link;
  305.             history[key].requester = requester_name;
  306.             history[key].title = title;
  307.             history[key].reward = reward;
  308.             history[key].hits = hits;
  309.             history[key].rid = requester_id;
  310.             history[key].acc_link = accept_link;
  311. 			HITStorage.indexedDB.checkRequester(requester_id,key);
  312. 			HITStorage.indexedDB.checkTitle(title,key);
  313.             if (searched_once)
  314.             {
  315.                 history[key].initial_time = new Date().getTime();//-1000*(save_new_results_time - SEARCH_REFRESH);
  316.                 history[key].new_result = 0;
  317.             }
  318.             else
  319.             {
  320.                 history[key].initial_time = new Date().getTime()-1000*save_new_results_time;
  321.                 history[key].new_result = 1000*save_new_results_time;
  322.             }
  323.             if (not_qualified_group_IDs.indexOf(group_ID)!==-1)
  324.                 history[key].qualified = false;
  325.             else
  326.                 history[key].qualified = true;
  327.  
  328.             history[key].found_this_time = true;
  329.         }
  330.         else
  331.         {
  332.             history[key].new_result = new Date().getTime() - history[key].initial_time;
  333.             history[key].found_this_time = true;
  334.             history[key].hits = hits;
  335.         }
  336.     }
  337. }
  338.  
  339. function statusdetail_loop(next_URL)
  340. {
  341.     if (global_run == true)
  342.     {
  343.         if (next_URL.length != 0)
  344.         {
  345.             $.get(next_URL, function(data)
  346.             {
  347.                 var $src = $(data);
  348.                 var maxpagerate = $src.find('td[class="error_title"]:contains("You have exceeded the maximum allowed page request rate for this website.")');
  349.                 if (maxpagerate.length == 0)
  350.                 {
  351.                     set_progress_report("Processing page " + next_page, false);
  352.                     scrape($src);
  353.         
  354.                     $next_URL = $src.find('a[href^="/mturk/viewsearchbar"]:contains("Next")');
  355.                     next_URL = ($next_URL.length != 0) ? $next_URL.attr("href") : "";
  356.                     next_page++;
  357.                     if (default_type == 1)
  358.                     {
  359.                         var hmin = MINIMUM_HITS+1;
  360.                         for (j = 0; j < found_key_list.length; j++)
  361.                         {
  362.                             if (history[found_key_list[j]].hits < hmin)
  363.                             {
  364.                                 next_URL = "";
  365.                                 next_page = -1;
  366.                                 break;
  367.                             }
  368.                         }
  369.                     }
  370.         
  371.                     else if (next_page > PAGES_TO_SCRAPE && correct_for_skips)
  372.                     {
  373.                         var skipped_hits = 0;
  374.                         var added_pages = 0;
  375.                         for (j = 0; j < found_key_list.length; j++)
  376.                         {
  377.                             var obj = history[found_key_list[j]];
  378.                             if (! ignore_check(obj.requester,obj.title))
  379.                                 skipped_hits++;
  380.                         }
  381.                         added_pages = Math.floor(skipped_hits/10);
  382.                         if (skipped_hits%10 >6)
  383.                             added_pages++;
  384.                         if (next_page > PAGES_TO_SCRAPE + added_pages)
  385.                         {
  386.                             next_URL = "";
  387.                             next_page = -1;
  388.                         }
  389.             
  390.                     }
  391.                     else if (next_page > PAGES_TO_SCRAPE)
  392.                     {
  393.                         next_URL = "";
  394.                         next_page = -1;
  395.                     }
  396.                     
  397.                     setTimeout(function(){statusdetail_loop(next_URL);}, STATUSDETAIL_DELAY);
  398.                 }
  399.                 else
  400.                 {
  401.                     console.log("MPRE");
  402.                     setTimeout(function(){statusdetail_loop(next_URL);}, MPRE_DELAY);
  403.                 }
  404.             });
  405.         }
  406.         else
  407.         {
  408.             searched_once = true;
  409.             var found_hits = found_key_list.length;
  410.             var shown_hits = 0;
  411.             var new_hits = 0;
  412.             var url = API_MULTI_ATTRS_URL;
  413.             var rids = [];
  414.             var lastRow = text_area.rows.length - 1;
  415.             for (i = lastRow; i>0; i--)
  416.                 text_area.deleteRow(i);
  417.             for (j = 0; j < found_key_list.length; j++)
  418.             {
  419.                 var obj = history[found_key_list[j]];
  420.                 if (ignore_check(obj.requester,obj.title) && obj.found_this_time){
  421.                     ++shown_hits;
  422.                     var col_heads = ["<a href='"+ LINK_BASE+obj.req_link +"' target='_blank'>" + obj.requester + "</a>","<a href='"+ LINK_BASE+obj.prev_link +"' target='_blank'>" + obj.title + "</a>",obj.reward,obj.hits,"TO down","<a href='"+ LINK_BASE+obj.acc_link +"' target='_blank'>Accept</a>"];
  423.                     var row = text_area.insertRow(text_area.rows.length);
  424.                     url += obj.rid + ',';
  425.                     rids.push(obj.rid);
  426. 					if (check_hitDB)
  427. 					{
  428. 						col_heads.push("R");
  429. 						col_heads.push("T");
  430. 					}
  431.                     if (!obj.qualified)
  432.                     {
  433.                         col_heads.push("Not Qualified");
  434.                     }
  435.                     for (i=0; i<col_heads.length; i++)
  436.                     {
  437.                         var this_cell = row.insertCell(i);
  438. 						row.cells[i].style.fontSize = default_text_size;
  439.                         this_cell.innerHTML = col_heads[i];
  440.                         if(i>1)
  441.                             this_cell.style.textAlign = 'center';
  442. 						if (check_hitDB)
  443. 						{
  444. 							if (i==6)
  445. 							{
  446. 								if (obj.reqdb)
  447. 									this_cell.style.backgroundColor = GREEN;
  448. 								else
  449. 									this_cell.style.backgroundColor = RED;
  450. 							}
  451. 							else if (i==7)
  452. 							{
  453. 								if (obj.titledb)
  454. 									this_cell.style.backgroundColor = GREEN;
  455. 								else
  456. 									this_cell.style.backgroundColor = RED;
  457. 							}
  458. 							else if (i==8)
  459. 								this_cell.style.backgroundColor = DARKGREY;
  460. 						}
  461. 						else if (i==6)
  462. 							this_cell.style.backgroundColor = DARKGREY;
  463.                     }
  464.                     if (Object.keys(history).length>0)
  465.                     {
  466.                         if (obj.new_result < 1000*save_new_results_time)
  467.                         {
  468.                             new_hits++;
  469.                             for (i in col_heads)
  470.                             {
  471.                                 row.cells[i].style.fontSize = default_text_size + 1;
  472.                                 row.cells[i].style.fontWeight = "bold";
  473.                             }
  474.                         }
  475.                     }
  476.                 }
  477.             }
  478.             set_progress_report("Scrape complete. " + shown_hits + " HITs found (" + new_hits + " new results). " + (found_hits - shown_hits) + " HITs ignored.", false);
  479.             url = url.substring(0,url.length - 1);
  480.             var success_flag = false;
  481.             GM_xmlhttpRequest(
  482.             {
  483.                 method: "GET",
  484.                 url: url,
  485.                 onload: function (results)
  486.                 {
  487.                     rdata = $.parseJSON(results.responseText);
  488.                     for (i = 0; i < rids.length; i++)
  489.                     {
  490.                         text_area.rows[i+1].style.backgroundColor = GREY;
  491.                         if (rdata[rids[i]])
  492.                         {
  493.                             var pay = rdata[rids[i]].attrs.pay
  494.                             var reviews = rdata[rids[i]].reviews
  495.                             var average = 0;
  496.                             var sum = 0;
  497.                             var divisor = 0;
  498.                             var comm = rdata[rids[i]].attrs.comm;
  499.                             var fair = rdata[rids[i]].attrs.fair;
  500.                             var fast = rdata[rids[i]].attrs.fast;
  501.                             if (comm > 0)
  502.                             {
  503.                                 sum += COMM_WEIGHT*comm;
  504.                                 divisor += COMM_WEIGHT;
  505.                             }
  506.                             if (pay > 0)
  507.                             {
  508.                                 sum += PAY_WEIGHT*pay;
  509.                                 divisor += PAY_WEIGHT;
  510.                             }
  511.                             if (fair > 0)
  512.                             {
  513.                                 sum += FAIR_WEIGHT*fair;
  514.                                 divisor += FAIR_WEIGHT;
  515.                             }
  516.                             if (fast > 0)
  517.                             {
  518.                                 sum += FAST_WEIGHT*fast;
  519.                                 divisor += FAST_WEIGHT;
  520.                             }
  521.                             if (divisor > 0)
  522.                             {
  523.                                 average = sum/divisor;
  524.                             }
  525.                             text_area.rows[i+1].cells[4].innerHTML = "<a href='"+ TO_REQ_URL+rids[i] +"' target='_blank'>" + pay + "</a>";
  526.                             if (reviews > 4)
  527.                             {
  528.                                 if (average > 4.49)
  529.                                     text_area.rows[i+1].style.backgroundColor = GREEN;
  530.                                 else if (average > 3.49)
  531.                                     text_area.rows[i+1].style.backgroundColor = LIGHTGREEN;
  532.                                 //else if (average > 2.99)
  533.                                  //   text_area.rows[i+1].style.backgroundColor = YELLOW;
  534.                                 else if (average > 1.99)
  535.                                     text_area.rows[i+1].style.backgroundColor = ORANGE;
  536.                                 else if (average > 0)
  537.                                     text_area.rows[i+1].style.backgroundColor = RED;
  538.                              }
  539.                         }
  540.                         else
  541.                         {
  542.                             text_area.rows[i+1].cells[4].innerHTML = "No data";
  543.                         }
  544.                     }
  545.                     success_flag = true;
  546.                  }
  547.             });
  548.             if (!success_flag)
  549.                 for (i = 0; i < rids.length; i++) text_area.rows[i+1].style.backgroundColor = GREY;
  550.             
  551.             statusdetail_loop_finished = true;
  552.             if (SEARCH_REFRESH>0)
  553.             {
  554.                 wait_loop = setTimeout(function(){if (global_run) start_it();}, 1000*SEARCH_REFRESH);
  555.                 display_wait_time(SEARCH_REFRESH);
  556.             }
  557.             else
  558.             {
  559.                 global_run = false;
  560.                 big_red_button.textContent = "Start";
  561.             }
  562.         }
  563.     }
  564. }
  565.  
  566. function ignore_check(r,t){
  567.     if (ignore_list.indexOf(r)==-1)
  568.     {
  569.         return true;
  570.     }
  571.     return false;
  572. }
  573.  
  574. function start_running()
  575. {
  576.     if (big_red_button.textContent == "Start")
  577.     {
  578.         global_run = true;
  579.         initial_url = URL_BASE;
  580.         if (search_input.value.length>0)
  581.         {
  582.             initial_url = initial_url.replace("searchWords=", "searchWords=" + search_input.value);
  583.         }
  584.         if (time_input.value.replace(/[^0-9]+/g,"") != "")
  585.         {
  586.             SEARCH_REFRESH = Number(time_input.value);
  587.         }
  588.         if (page_input.value.replace(/[^0-9]+/g,"") != "")
  589.         {
  590.             PAGES_TO_SCRAPE = Number(page_input.value);
  591.         }
  592.         if (min_input.value.replace(/[^0-9]+/g,"") != "")
  593.         {
  594.             MINIMUM_HITS = Number(min_input.value);
  595.         }
  596.         if (new_time_display_input.value.replace(/[^0-9]+/g,"") != "")
  597.         {
  598.             save_new_results_time = Number(new_time_display_input.value);
  599.         }
  600.         if (reward_input.value.replace(/[^0-9]+/g,"") != "")
  601.         {
  602.             initial_url += "&minReward=" + reward_input.value;
  603.         }
  604.         else
  605.         {
  606.             initial_url += "&minReward=0.00";
  607.         }
  608.         if (qual_input.checked)
  609.         {
  610.             initial_url += "&qualifiedFor=on"
  611.         }
  612.         else
  613.         {
  614.             initial_url += "&qualifiedFor=off"
  615.         }
  616. 		if (masters_input.checked)
  617.         {
  618.             initial_url += "&requiresMasterQual=on"
  619.         }
  620.         if (sort_input1.checked)
  621.         {
  622.             initial_url+= "&sortType=LastUpdatedTime%3A1";
  623.             default_type = 0;
  624.         }
  625.         else if (sort_input2.checked)
  626.         {
  627.             initial_url+= "&sortType=NumHITs%3A1";
  628.             default_type = 1;
  629.         }
  630.         else if (sort_input3.checked)
  631.         {
  632.             initial_url+= "&sortType=Reward%3A1";
  633.             default_type = 0;
  634.         }
  635.         
  636.         initial_url+="&pageNumber=1&searchSpec=HITGroupSearch"
  637.         start_it();
  638.     }
  639.     else
  640.     {
  641.         global_run = false;
  642.         clearTimeout(wait_loop);
  643.         big_red_button.textContent = "Start";
  644.         set_progress_report("Stopped", true);
  645.     }
  646. }
  647.  
  648. function start_it()
  649. {
  650.     statusdetail_loop_finished = false;
  651.     big_red_button.textContent = "Stop";
  652.     found_key_list=[];
  653.     var ctime = new Date().getTime()
  654.     if (ctime - last_clear_time > save_results_time*666)
  655.     {
  656.         var last_history=history;
  657.         history = {};
  658.         for (var key in last_history)
  659.         {
  660.             if (last_history[key].new_result<save_results_time*1000)
  661.             {
  662.                 history[key]=last_history[key];
  663.                 if (last_history[key].found_this_time)
  664.                 {
  665.                     last_history[key].found_this_time = false;
  666.                     if (last_history[key].new_result>save_new_results_time*1000)
  667.                         last_history[key].initial_time = ctime-1000*save_new_results_time;
  668.                 }
  669.             }
  670.  
  671.         }
  672.         last_clear_time = ctime;
  673.     }
  674.     next_page = 1;
  675.     statusdetail_loop(initial_url);
  676. }
  677.  
  678.  
  679. function show_interface()
  680. {
  681.     control_panel.style.color = BROWN;
  682.     control_panel.style.fontSize = 14;
  683.     control_panel.removeChild(big_red_button);
  684.     control_panel.appendChild(document.createTextNode("Auto-refresh delay: "));
  685.     time_input.onkeydown = function(event){if (event.keyCode == 13){start_running();}};
  686.     time_input.title = "Enter search refresh delay in seconds\n" + "Enter 0 for no auto-refresh\n" + "Default is 0 (no auto-refresh)";
  687.     time_input.size = 3;
  688.     control_panel.appendChild(time_input);
  689.     
  690.     control_panel.appendChild(document.createTextNode("   "));
  691.     
  692.     control_panel.appendChild(document.createTextNode("Pages to scrape: "));
  693.     page_input.onkeydown = function(event){if (event.keyCode == 13){start_running();}};
  694.     page_input.title = "Enter number of pages to scrape\n" + "Default is 4";
  695.     page_input.size = 3;
  696.     control_panel.appendChild(page_input);
  697.     
  698.     control_panel.appendChild(document.createTextNode("   "));
  699.     
  700.     control_panel.appendChild(document.createTextNode("Minimum batch size: "));
  701.     min_input.onkeydown = function(event){if (event.keyCode == 13){start_running();}};
  702.     min_input.title = "Enter minimum HITs for batch search\n" + "Default is 100";
  703.     min_input.size = 3;
  704.     control_panel.appendChild(min_input);
  705.     control_panel.appendChild(document.createTextNode("   "));
  706.     
  707.     control_panel.appendChild(document.createTextNode("New HIT highlighting: "));
  708.     new_time_display_input.onkeydown = function(event){if (event.keyCode == 13){start_running();}};
  709.     new_time_display_input.title = "Enter time (in seconds) to keep new HITs highlighted\n" + "Default is 300 (5 minutes)";
  710.     new_time_display_input.size = 6;
  711.     control_panel.appendChild(new_time_display_input);
  712.     
  713.     control_panel.appendChild(document.createElement("P"));
  714.     control_panel.appendChild(document.createTextNode("Minimum reward: "));
  715.     reward_input.size = 6;
  716.     control_panel.appendChild(reward_input);
  717.     control_panel.appendChild(document.createTextNode("   "));
  718.  
  719.     control_panel.appendChild(document.createTextNode("Qualified"));
  720.     control_panel.appendChild(qual_input);
  721.     control_panel.appendChild(document.createTextNode("     "));
  722. 	control_panel.appendChild(document.createTextNode("Masters"));
  723.     control_panel.appendChild(masters_input);
  724.     control_panel.appendChild(document.createTextNode("     "));
  725.     control_panel.appendChild(document.createTextNode("Sort types:   "));
  726.     control_panel.appendChild(sort_input1);
  727.     control_panel.appendChild(document.createTextNode("Latest"));
  728.     control_panel.appendChild(sort_input2);
  729.     control_panel.appendChild(document.createTextNode("Most Available"));
  730.     control_panel.appendChild(sort_input3);
  731.     control_panel.appendChild(document.createTextNode("Amount"));
  732.        
  733.     control_panel.appendChild(document.createElement("P"));
  734.     
  735.     control_panel.appendChild(search_input);
  736.     search_input.size = 20;
  737.     search_input.title = "Enter a search term to include\n" + "Default is blank (no included terms)";
  738.     search_input.placeholder="Enter search terms here";
  739.     
  740.     control_panel.appendChild(document.createTextNode("   "));
  741.     
  742.     big_red_button.textContent = "Start";
  743.     big_red_button.onclick = function(){start_running();};
  744.     
  745.     control_panel.appendChild(big_red_button);
  746.    
  747.     control_panel.appendChild(document.createTextNode("   "));
  748.     control_panel.appendChild(progress_report);
  749.      
  750.     control_panel.appendChild(document.createElement("P"));
  751.     
  752.     text_area.style.fontWeight = 400;
  753.     text_area.createCaption().innerHTML = "HITs";
  754.     var col_heads = ['Requester','Title','Reward','HITs Available','TO pay',"Accept HIT"];
  755.     var row = text_area.createTHead().insertRow(0);
  756.     text_area.caption.style.fontWeight = 800;
  757.     text_area.caption.style.color = BROWN;
  758. 	if (default_text_size > 10)
  759. 		text_area.cellPadding=Math.min(Math.max(1,Math.floor((default_text_size-10)/2)),5);
  760.     console.log(text_area.cellPadding);
  761.     //text_area.cellPadding=2;
  762.     text_area.caption.style.fontSize = 28;
  763.     text_area.rows[0].style.fontWeight = 800;
  764.     text_area.rows[0].style.color = BROWN;
  765.     for (i=0; i<col_heads.length; i++)
  766.     {
  767.         var this_cell = row.insertCell(i);
  768.         this_cell.innerHTML = col_heads[i];
  769.         this_cell.style.fontSize = 14;
  770.         if (i > 1)
  771.             this_cell.style.textAlign = 'center';
  772.     }
  773.     
  774.     control_panel.appendChild(text_area);
  775. }

欢迎访问Greasy Fork镜像

土豆服务器，请按需使用

镜像地址随时可能被墙，建议加群获取最新地址

加入Q群

QingJ © 2025

镜像随时可能失效，请加Q群300939539或关注我们的公众号极客氢云获取最新地址