lib 1400208 // tika - 50 MB! import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.parser.AutoDetectParser; import org.apache.tika.parser.ParseContext; import org.apache.tika.sax.BodyContentHandler; import org.apache.http.util.*; import org.apache.http.protocol.HTTP; static Map tika(String url) { new DefaultHttpClient httpclient; //HttpClient httpclient = apacheHttp_trustingClient(); Map map = new HashMap(); pcall { HttpGet httpGet = new HttpGet(url); HttpResponse response = httpclient.execute(httpGet); HttpEntity entity = response.getEntity(); InputStream input = null; if (entity != null) { try { input = entity.getContent(); //S inputString = EntityUtils.toString(entity, HTTP.UTF_8); //input = new ByteArrayInputStream(toUTF8(inputString)); //new StringWriter stringWriter; //BodyContentHandler handler = new(stringWriter); new BodyContentHandler handler; //new ByteArrayOutputStream outputStream; //BodyContentHandler handler = new(outputStream); new Metadata metadata; new AutoDetectParser parser; new ParseContext parseContext; parser.parse(input, handler, metadata, parseContext); map.put("text", handler.toString() /*.replaceAll("\n|\r|\t", " ") */); //map.put("text", stringWriter.toString()); //map.put("text", fromUTF8(outputStream.toByteArray())); //map.put("text", fromUTF8(charsToSingleBytes(handler.toString()))); map.put("title", metadata.get(TikaCoreProperties.TITLE)); map.put("pageCount", metadata.get("xmpTPg:NPages")); map.put("status_code", response.getStatusLine().getStatusCode() + ""); } catch (Exception e) { e.printStackTrace(); } finally { if (input != null) input.close(); } } } ret map; }