Changeset 9302

Show
Ignore:
Timestamp:
05/14/08 20:19:36 (2 months ago)
Author:
dfhuynh
Message:

Added scrape referrers task.

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved
  • collector/trunk/src/main/java/IndexingUtilities.java

    r9299 r9302  
    4747        } 
    4848 
    49         static public String encode(String s) { 
    50             try { 
    51                 return IndexingUtilities.codec.encode(s, IndexingUtilities.URL_ENCODING); 
    52             } catch (Exception e) { 
    53                 throw new RuntimeException("Exception encoding " + s + " with " + IndexingUtilities.URL_ENCODING + " encoding."); 
    54             } 
    55         } 
    56  
    5749        static boolean checkWithGoogle(String urlString) { 
    5850            String googleUrlString = "http://www.google.com/search?q=" + 
     
    8375        static final String URL_ENCODING = "UTF-8"; 
    8476 
    85 
     77        static public String encode(String s) { 
     78            try { 
     79                return IndexingUtilities.codec.encode(s, IndexingUtilities.URL_ENCODING); 
     80            } catch (Exception e) { 
     81                throw new RuntimeException("Exception encoding " + s + " with " + IndexingUtilities.URL_ENCODING + " encoding."); 
     82            } 
     83        } 
     84 
     85    static public String encodeFile(String s) { 
     86        s = encode(s.replace(':', '_').replace('/', '_').replace('?', '_').replace('&', '_')).replace('%', '_'); 
     87        if (s.length() > 128) { 
     88                s = s.substring(0, 128) + ((long) (Math.random() * 1000000)); 
     89        } 
     90        return s; 
     91    }} 
  • collector/trunk/src/main/java/TestIndexedReferrersWithoutQueries.java

    r9301 r9302  
    3434                    } 
    3535                     
    36                     if (n > 1500) { // skip column headings 
     36                    if (n > 2715) { // skip column headings 
    3737                        boolean didQueries = processLine(n, line, writer); 
    3838                        writer.flush();