Description
I found some sites, where the header says: "Content-Type: text/html; charset=". This causes an exception in the HtmlParser. My suggestion:
Index: src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java
===================================================================
— src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java (revision 279397)
+++ src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java (working copy)
@@ -120,7 +120,7 @@
byte[] contentInOctets = content.getContent();
InputSource input = new InputSource(new ByteArrayInputStream(contentInOctets));
String encoding = StringUtil.parseCharacterEncoding(contentType);
- if (encoding!=null) {
+ if (encoding!=null && !"".equals(encoding)) {
metadata.put("OriginalCharEncoding", encoding);
if ((encoding = StringUtil.resolveEncodingAlias(encoding)) != null) {
metadata.put("CharEncodingForConversion", encoding);