Index: lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/AnalyzerProfile.java
===================================================================
--- lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/AnalyzerProfile.java (revision 1502533)
+++ lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/AnalyzerProfile.java (working copy)
@@ -20,8 +20,11 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.util.Properties;
+import org.apache.lucene.util.IOUtils;
+
/**
* Manages analysis data configuration for SmartChineseAnalyzer
*
@@ -77,13 +80,13 @@
Properties prop = new Properties();
try {
FileInputStream input = new FileInputStream(propFile);
- prop.load(input);
+ prop.load(new InputStreamReader(input, IOUtils.CHARSET_UTF_8));
String dir = prop.getProperty("analysis.data.dir", "");
input.close();
return dir;
} catch (IOException e) {
+ return "";
}
- return "";
}
}
Index: lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java
===================================================================
--- lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java (revision 1502533)
+++ lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java (working copy)
@@ -18,9 +18,9 @@
*/
import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.Reader;
+import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -80,8 +80,7 @@
}
// read props from string
this.props = new Properties();
- // props.load always assumes iso8859-1...
- props.load(new ByteArrayInputStream(sb.toString().getBytes("ISO-8859-1")));
+ props.load(new StringReader(sb.toString()));
// make sure work dir is set properly
if (props.get("work.dir") == null) {
Index: lucene/tools/forbiddenApis/base.txt
===================================================================
--- lucene/tools/forbiddenApis/base.txt (revision 1502533)
+++ lucene/tools/forbiddenApis/base.txt (working copy)
@@ -22,9 +22,10 @@
java.util.concurrent.Executors#defaultThreadFactory()
java.util.concurrent.Executors#privilegedThreadFactory()
-@defaultMessage Properties files should be read/written with InputStream/OutputStream for maximum compatibility, as it uses the official "properties file format" with unicode escapes and properly defined encoding
-java.util.Properties#load(java.io.Reader)
-java.util.Properties#store(java.io.Writer,java.lang.String)
+@defaultMessage Properties files should be read/written with Reader/Writer, using a charset of UTF-8. This allows reading older files with unicode escapes, too.
+java.util.Properties#load(java.io.InputStream)
+java.util.Properties#save(java.io.OutputStream,java.lang.String)
+java.util.Properties#store(java.io.OutputStream,java.lang.String)
java.lang.Character#codePointBefore(char[],int) @ Implicit start offset is error-prone when the char[] is a buffer and the first chars are random chars
java.lang.Character#codePointAt(char[],int) @ Implicit end offset is error-prone when the char[] is a buffer and the last chars are random chars
Index: solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java
===================================================================
--- solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java (revision 1502533)
+++ solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SimplePropertiesWriter.java (working copy)
@@ -22,9 +22,10 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
-import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
@@ -33,6 +34,7 @@
import java.util.Map;
import java.util.Properties;
+import org.apache.lucene.util.IOUtils;
import org.apache.solr.core.SolrCore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -179,7 +181,7 @@
@Override
public void persist(Map propObjs) {
- OutputStream propOutput = null;
+ Writer propOutput = null;
Properties existingProps = mapToProperties(readIndexerProperties());
Properties newProps = mapToProperties(propObjs);
try {
@@ -189,18 +191,14 @@
filePath += File.separator;
}
filePath += filename;
- propOutput = new FileOutputStream(filePath);
+ propOutput = new OutputStreamWriter(new FileOutputStream(filePath), IOUtils.CHARSET_UTF_8);
existingProps.store(propOutput, null);
log.info("Wrote last indexed time to " + filename);
} catch (Exception e) {
throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
"Unable to persist Index Start Time", e);
} finally {
- try {
- if (propOutput != null) propOutput.close();
- } catch (IOException e) {
- propOutput = null;
- }
+ IOUtils.closeWhileHandlingException(propOutput);
}
}
@@ -215,16 +213,12 @@
}
filePath += filename;
propInput = new FileInputStream(filePath);
- props.load(propInput);
+ props.load(new InputStreamReader(propInput, IOUtils.CHARSET_UTF_8));
log.info("Read " + filename);
} catch (Exception e) {
log.warn("Unable to read: " + filename);
} finally {
- try {
- if (propInput != null) propInput.close();
- } catch (IOException e) {
- propInput = null;
- }
+ IOUtils.closeWhileHandlingException(propInput);
}
return propertiesToMap(props);
}
Index: solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java
===================================================================
--- solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java (revision 1502533)
+++ solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ZKPropertiesWriter.java (working copy)
@@ -16,11 +16,12 @@
*/
package org.apache.solr.handler.dataimport;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
+import java.io.StringReader;
+import java.io.StringWriter;
import java.util.Map;
import java.util.Properties;
+import org.apache.lucene.util.IOUtils;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.zookeeper.KeeperException.NodeExistsException;
import org.slf4j.Logger;
@@ -63,10 +64,10 @@
public void persist(Map propObjs) {
Properties existing = mapToProperties(readIndexerProperties());
existing.putAll(mapToProperties(propObjs));
- ByteArrayOutputStream output = new ByteArrayOutputStream();
+ StringWriter output = new StringWriter();
try {
existing.store(output, "");
- byte[] bytes = output.toByteArray();
+ byte[] bytes = output.toString().getBytes(IOUtils.CHARSET_UTF_8);
if (!zkClient.exists(path, false)) {
try {
zkClient.makePath(path, false);
@@ -89,8 +90,7 @@
try {
byte[] data = zkClient.getData(path, null, null, false);
if (data != null) {
- ByteArrayInputStream input = new ByteArrayInputStream(data);
- props.load(input);
+ props.load(new StringReader(new String(data, "UTF-8")));
}
} catch (Throwable e) {
log.warn(
Index: solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java
===================================================================
--- solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java (revision 1502533)
+++ solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java (working copy)
@@ -17,6 +17,7 @@
package org.apache.solr.response;
+import org.apache.lucene.util.IOUtils;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.SolrResponseBase;
@@ -150,7 +151,7 @@
try {
is = resourceLoader.getResourceStream(propFile);
Properties props = new Properties();
- props.load(is);
+ props.load(new InputStreamReader(is, IOUtils.CHARSET_UTF_8));
engine.init(props);
}
finally {
Index: solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java
===================================================================
--- solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java (revision 1502533)
+++ solr/core/src/java/org/apache/solr/cloud/SolrZkServer.java (working copy)
@@ -179,7 +179,7 @@
Properties cfg = new Properties();
FileInputStream in = new FileInputStream(configFile);
try {
- cfg.load(in);
+ cfg.load(new InputStreamReader(in, IOUtils.CHARSET_UTF_8));
} finally {
in.close();
}
Index: solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
===================================================================
--- solr/core/src/java/org/apache/solr/core/CoreDescriptor.java (revision 1502533)
+++ solr/core/src/java/org/apache/solr/core/CoreDescriptor.java (working copy)
@@ -22,11 +22,13 @@
import org.apache.commons.lang.StringUtils;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.common.SolrException;
+import org.apache.solr.util.IOUtils;
import org.apache.solr.util.PropertiesUtil;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.util.Locale;
import java.util.Properties;
@@ -183,15 +185,18 @@
String filename = coreProperties.getProperty(CORE_PROPERTIES, DEFAULT_EXTERNAL_PROPERTIES_FILE);
File propertiesFile = resolvePaths(filename);
if (propertiesFile.exists()) {
+ FileInputStream in = null;
try {
+ in = new FileInputStream(propertiesFile);
Properties externalProps = new Properties();
- externalProps.load(new FileInputStream(propertiesFile));
+ externalProps.load(new InputStreamReader(in, "UTF-8"));
coreProperties.putAll(externalProps);
- }
- catch (IOException e) {
+ } catch (IOException e) {
String message = String.format(Locale.ROOT, "Could not load properties from %s: %s:",
propertiesFile.getAbsoluteFile(), e.toString());
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, message);
+ } finally {
+ IOUtils.closeQuietly(in);
}
}
}
Index: solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
===================================================================
--- solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java (revision 1502533)
+++ solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java (working copy)
@@ -17,6 +17,7 @@
* limitations under the License.
*/
+import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import org.apache.solr.common.SolrException;
import org.apache.solr.util.IOUtils;
@@ -27,7 +28,9 @@
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.OutputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
import java.util.Date;
import java.util.List;
import java.util.Properties;
@@ -73,9 +76,9 @@
private void writePropertiesFile(CoreDescriptor cd, File propfile) {
Properties p = buildCoreProperties(cd);
- OutputStream os = null;
+ Writer os = null;
try {
- os = new FileOutputStream(propfile);
+ os = new OutputStreamWriter(new FileOutputStream(propfile), Charsets.UTF_8);
p.store(os, "Written by CorePropertiesLocator on " + new Date());
}
catch (IOException e) {
@@ -134,7 +137,7 @@
File instanceDir = propertiesFile.getParentFile();
Properties coreProperties = new Properties();
fis = new FileInputStream(propertiesFile);
- coreProperties.load(fis);
+ coreProperties.load(new InputStreamReader(fis, Charsets.UTF_8));
String name = createName(coreProperties, instanceDir);
return new CoreDescriptor(cc, name, instanceDir.getAbsolutePath(), coreProperties);
}
Index: solr/core/src/java/org/apache/solr/core/SolrCore.java
===================================================================
--- solr/core/src/java/org/apache/solr/core/SolrCore.java (revision 1502533)
+++ solr/core/src/java/org/apache/solr/core/SolrCore.java (working copy)
@@ -96,6 +96,7 @@
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.io.Writer;
import java.lang.reflect.Constructor;
import java.net.URL;
@@ -252,7 +253,7 @@
final InputStream is = new PropertiesInputStream(input);
try {
- p.load(is);
+ p.load(new InputStreamReader(is, "UTF-8"));
String s = p.getProperty("index");
if (s != null && s.trim().length() > 0) {
Index: solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
===================================================================
--- solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (revision 1502533)
+++ solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (working copy)
@@ -20,6 +20,7 @@
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.Writer;
import java.nio.ByteBuffer;
@@ -48,6 +49,7 @@
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import static org.apache.lucene.util.IOUtils.CHARSET_UTF_8;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
@@ -813,7 +815,7 @@
try {
final InputStream is = new PropertiesInputStream(input);
Properties props = new Properties();
- props.load(is);
+ props.load(new InputStreamReader(is, CHARSET_UTF_8));
return props;
} finally {
input.close();
Index: solr/core/src/java/org/apache/solr/handler/SnapPuller.java
===================================================================
--- solr/core/src/java/org/apache/solr/handler/SnapPuller.java (revision 1502533)
+++ solr/core/src/java/org/apache/solr/handler/SnapPuller.java (working copy)
@@ -40,7 +40,9 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.text.SimpleDateFormat;
@@ -73,6 +75,7 @@
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import static org.apache.lucene.util.IOUtils.CHARSET_UTF_8;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
@@ -578,7 +581,7 @@
}
final IndexOutput out = dir.createOutput(REPLICATION_PROPERTIES, DirectoryFactory.IOCONTEXT_NO_CACHE);
- OutputStream outFile = new PropertiesOutputStream(out);
+ Writer outFile = new OutputStreamWriter(new PropertiesOutputStream(out), CHARSET_UTF_8);
try {
props.store(outFile, "Replication details");
dir.sync(Collections.singleton(REPLICATION_PROPERTIES));
@@ -890,7 +893,7 @@
final InputStream is = new PropertiesInputStream(input);
try {
- p.load(is);
+ p.load(new InputStreamReader(is, CHARSET_UTF_8));
} catch (Exception e) {
LOG.error("Unable to load " + SnapPuller.INDEX_PROPERTIES, e);
} finally {
@@ -904,9 +907,9 @@
}
final IndexOutput out = dir.createOutput(SnapPuller.INDEX_PROPERTIES, DirectoryFactory.IOCONTEXT_NO_CACHE);
p.put("index", tmpIdxDirName);
- OutputStream os = null;
+ Writer os = null;
try {
- os = new PropertiesOutputStream(out);
+ os = new OutputStreamWriter(new PropertiesOutputStream(out), CHARSET_UTF_8);
p.store(os, SnapPuller.INDEX_PROPERTIES);
dir.sync(Collections.singleton(INDEX_PROPERTIES));
} catch (Exception e) {
Index: solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java
===================================================================
--- solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java (revision 1502533)
+++ solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java (working copy)
@@ -19,6 +19,8 @@
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
import java.util.Properties;
import javax.xml.parsers.ParserConfigurationException;
@@ -30,6 +32,7 @@
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
@@ -97,15 +100,15 @@
File newDir = new File(h.getCore().getDataDir() + "index_temp");
newDir.mkdirs();
p.put("index", newDir.getName());
- FileOutputStream os = null;
+ Writer os = null;
try {
- os = new FileOutputStream(idxprops);
+ os = new OutputStreamWriter(new FileOutputStream(idxprops), IOUtils.CHARSET_UTF_8);
p.store(os, "index properties");
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unable to write " + SnapPuller.INDEX_PROPERTIES, e);
} finally {
- if (os != null) os.close();
+ IOUtils.closeWhileHandlingException(os);
}
//add a doc in the new index dir
Index: solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java
===================================================================
--- solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java (revision 1502533)
+++ solr/core/src/test/org/apache/solr/core/TestCoreDiscovery.java (working copy)
@@ -27,6 +27,8 @@
import java.io.File;
import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
import java.util.Properties;
public class TestCoreDiscovery extends SolrTestCaseJ4 {
@@ -78,7 +80,7 @@
private void addCoreWithProps(Properties stockProps, File propFile) throws Exception {
if (!propFile.getParentFile().exists()) propFile.getParentFile().mkdirs();
- FileOutputStream out = new FileOutputStream(propFile);
+ Writer out = new OutputStreamWriter(new FileOutputStream(propFile), IOUtils.CHARSET_UTF_8);
try {
stockProps.store(out, null);
} finally {
Index: solr/core/src/test/org/apache/solr/handler/admin/CoreAdminCreateDiscoverTest.java
===================================================================
--- solr/core/src/test/org/apache/solr/handler/admin/CoreAdminCreateDiscoverTest.java (revision 1502533)
+++ solr/core/src/test/org/apache/solr/handler/admin/CoreAdminCreateDiscoverTest.java (working copy)
@@ -18,6 +18,7 @@
package org.apache.solr.handler.admin;
import org.apache.commons.io.FileUtils;
+import org.apache.lucene.util.IOUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CoreAdminParams;
@@ -30,6 +31,7 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.util.Properties;
public class CoreAdminCreateDiscoverTest extends SolrTestCaseJ4 {
@@ -112,7 +114,7 @@
File propFile = new File(solrHomeDirectory, coreSysProps + "/" + CorePropertiesLocator.PROPERTIES_FILENAME);
FileInputStream is = new FileInputStream(propFile);
try {
- props.load(is);
+ props.load(new InputStreamReader(is, IOUtils.CHARSET_UTF_8));
} finally {
org.apache.commons.io.IOUtils.closeQuietly(is);
}
@@ -217,7 +219,7 @@
File propFile = new File(solrHomeDirectory, coreNormal + "/" + CorePropertiesLocator.PROPERTIES_FILENAME);
FileInputStream is = new FileInputStream(propFile);
try {
- props.load(is);
+ props.load(new InputStreamReader(is, IOUtils.CHARSET_UTF_8));
} finally {
org.apache.commons.io.IOUtils.closeQuietly(is);
}
Index: solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java
===================================================================
--- solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java (revision 1502533)
+++ solr/core/src/test/org/apache/solr/TestSolrCoreProperties.java (working copy)
@@ -124,7 +124,7 @@
Properties p = new Properties();
p.setProperty("foo.foo1", "f1");
p.setProperty("foo.foo2", "f2");
- FileOutputStream fos = new FileOutputStream(confDir + File.separator + "solrcore.properties");
+ Writer fos = new OutputStreamWriter(new FileOutputStream(confDir + File.separator + "solrcore.properties"), IOUtils.CHARSET_UTF_8);
p.store(fos, null);
IOUtils.close(fos);
}