1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/python.core/src/org/netbeans/modules/python/api/PythonFileEncodingQuery.java Fri Jul 17 07:47:00 2015 +0200
1.3 @@ -0,0 +1,467 @@
1.4 +/*
1.5 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
1.6 + *
1.7 + * Copyright 2015 Oracle and/or its affiliates. All rights reserved.
1.8 + *
1.9 + * Oracle and Java are registered trademarks of Oracle and/or its affiliates.
1.10 + * Other names may be trademarks of their respective owners.
1.11 + *
1.12 + * The contents of this file are subject to the terms of either the GNU
1.13 + * General Public License Version 2 only ("GPL") or the Common
1.14 + * Development and Distribution License("CDDL") (collectively, the
1.15 + * "License"). You may not use this file except in compliance with the
1.16 + * License. You can obtain a copy of the License at
1.17 + * http://www.netbeans.org/cddl-gplv2.html
1.18 + * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
1.19 + * specific language governing permissions and limitations under the
1.20 + * License. When distributing the software, include this License Header
1.21 + * Notice in each file and include the License file at
1.22 + * nbbuild/licenses/CDDL-GPL-2-CP. Oracle designates this
1.23 + * particular file as subject to the "Classpath" exception as provided
1.24 + * by Oracle in the GPL Version 2 section of the License file that
1.25 + * accompanied this code. If applicable, add the following below the
1.26 + * License Header, with the fields enclosed by brackets [] replaced by
1.27 + * your own identifying information:
1.28 + * "Portions Copyrighted [year] [name of copyright owner]"
1.29 + *
1.30 + * If you wish your version of this file to be governed by only the CDDL
1.31 + * or only the GPL Version 2, indicate your decision by adding
1.32 + * "[Contributor] elects to include this software in this distribution
1.33 + * under the [CDDL or GPL Version 2] license." If you do not indicate a
1.34 + * single choice of license, a recipient has the option to distribute
1.35 + * your version of this file under either the CDDL, the GPL Version 2 or
1.36 + * to extend the choice of license to its licensees as provided above.
1.37 + * However, if you add GPL Version 2 code and therefore, elected the GPL
1.38 + * Version 2 license, then the option applies only if the new code is
1.39 + * made subject to such option by the copyright holder.
1.40 + *
1.41 + * Contributor(s):
1.42 + *
1.43 + * Portions Copyrighted 2015 Sun Microsystems, Inc.
1.44 + */
1.45 +package org.netbeans.modules.python.api;
1.46 +
1.47 +import java.io.ByteArrayInputStream;
1.48 +import java.io.IOException;
1.49 +import java.io.InputStream;
1.50 +import java.nio.ByteBuffer;
1.51 +import java.nio.CharBuffer;
1.52 +import java.nio.charset.Charset;
1.53 +import java.nio.charset.CharsetDecoder;
1.54 +import java.nio.charset.CharsetEncoder;
1.55 +import java.nio.charset.CoderResult;
1.56 +import java.nio.charset.IllegalCharsetNameException;
1.57 +import java.nio.charset.UnsupportedCharsetException;
1.58 +import java.util.HashMap;
1.59 +import java.util.Map;
1.60 +import java.util.Scanner;
1.61 +import java.util.logging.Level;
1.62 +import java.util.logging.Logger;
1.63 +import java.util.regex.Matcher;
1.64 +import java.util.regex.Pattern;
1.65 +import org.netbeans.api.queries.FileEncodingQuery;
1.66 +import org.netbeans.spi.queries.FileEncodingQueryImplementation;
1.67 +import org.openide.filesystems.FileObject;
1.68 +import org.openide.util.Exceptions;
1.69 +
1.70 +/**
1.71 + *
1.72 + * @author Vincent van der Leun
1.73 + */
1.74 +public class PythonFileEncodingQuery extends FileEncodingQueryImplementation {
1.75 +
1.76 + private final ThreadLocal<Boolean> callingFEQ = new ThreadLocal<Boolean>() {
1.77 + @Override
1.78 + protected Boolean initialValue() {
1.79 + return false;
1.80 + }
1.81 + };
1.82 +
1.83 + private static final Logger LOGGER = Logger.getLogger(PythonFileEncodingQuery.class.getName());
1.84 + private static final Pattern pattern = Pattern.compile("coding[:=]\\s*([-\\w.]+)"); // NOI18N
1.85 + private static final Map<String, String> encodingsAliases = new HashMap<>();
1.86 +
1.87 + private static final String ENCODING_ISO8859_1 = "ISO-8859-1"; //NOI18N
1.88 + private static final String ENCODING_UTF_8 = "utf-8"; //NOI18N
1.89 +
1.90 + static {
1.91 + // Convert encoding names only supported by Python to ones that are
1.92 + // supported by Python/Jython/Java.
1.93 + // I've only added the ones I believed are the most popular ones, so a
1.94 + // lot more must be added.
1.95 + // See https://docs.python.org/2/library/codecs.html
1.96 + encodingsAliases.put("utf_8", ENCODING_UTF_8); // NOI18N
1.97 + encodingsAliases.put("latin_1", ENCODING_ISO8859_1); // NOI18N
1.98 + encodingsAliases.put("latin-1", ENCODING_ISO8859_1); // NOI18N
1.99 + encodingsAliases.put("latin", ENCODING_ISO8859_1); // NOI18N
1.100 + encodingsAliases.put("latin1", ENCODING_ISO8859_1); // NOI18N
1.101 + encodingsAliases.put("8859", ENCODING_ISO8859_1); // NOI18N
1.102 + }
1.103 +
1.104 + // Can be called directly by code that just wants to know the encoding
1.105 + // as specified in the file header. FileEncodingQueryImplementation's
1.106 + // getEncoding() method forces the use of CharsetDecoder/CharsetEncoder.
1.107 + public String getPythonFileEncoding(final InputStream in) throws IOException {
1.108 + String encoding = null;
1.109 + try (Scanner sc = new Scanner(in)) {
1.110 + for (int i=0; i <= 1; i++) {
1.111 + if (sc.hasNextLine()) {
1.112 + String line = sc.nextLine();
1.113 + encoding = extractEncoding(line);
1.114 + if (encoding != null) {
1.115 + break;
1.116 + }
1.117 + }
1.118 + }
1.119 + }
1.120 + return recognizedEncoding(encoding);
1.121 + }
1.122 +
1.123 +
1.124 + // TODO: This is currently used by the Parser, but should ideally be removed.
1.125 + public String getPythonFileEncoding(String[] lines) {
1.126 + String encoding = null;
1.127 + for (int i=0; i <= (lines.length > 1 ? 1 : 0); i++) {
1.128 + String line = lines[i];
1.129 + encoding = extractEncoding(line);
1.130 + if (encoding != null) {
1.131 + break;
1.132 + }
1.133 + }
1.134 + return recognizedEncoding(encoding);
1.135 + }
1.136 +
1.137 +
1.138 + private String extractEncoding(String line) {
1.139 + if (line == null) {
1.140 + return null;
1.141 + }
1.142 + Matcher matcher = pattern.matcher(line);
1.143 + if (matcher.find()) {
1.144 + String res = matcher.group(1);
1.145 + return res;
1.146 + }
1.147 + return null;
1.148 + }
1.149 +
1.150 +
1.151 + private String recognizedEncoding(String inputEncoding) {
1.152 + // Returns the encoding if it's recognized by Charset.forName
1.153 + // method. When not recognized, an alternative for the specified
1.154 + // is returned if one is defined. Returns null when not recognized
1.155 + // at all.
1.156 +
1.157 + // PEP-0263 https://www.python.org/dev/peps/pep-0263/
1.158 + // suggests to silently ignore unrecognized encodings.
1.159 + String encoding = inputEncoding;
1.160 + if (encoding != null) {
1.161 + try {
1.162 + Charset temp = Charset.forName(encoding);
1.163 + LOGGER.log(Level.FINE, "Encoding recognized: {0}", encoding); //NOI18N
1.164 + } catch(Exception e) {
1.165 + // When using Python it makes sense to see if one of the
1.166 + // recognized aliases is used (name of codecs that are not
1.167 + // recognized by Java/Jython, but are recognized by Python)
1.168 + // There's probably a better way to do this.
1.169 + encoding = encodingsAliases.get(encoding);
1.170 + if (encoding != null) {
1.171 + LOGGER.log(Level.FINE, "Encoding alias: {0}", encoding); // NOI18N
1.172 + } else {
1.173 + LOGGER.log(Level.FINE, "Encoding not recognized"); // NOI18N
1.174 + }
1.175 + }
1.176 + }
1.177 + return encoding;
1.178 + }
1.179 +
1.180 +
1.181 + @Override
1.182 + public Charset getEncoding(FileObject file) {
1.183 + if(callingFEQ.get()) {
1.184 + //we are calling to the FEQ from within this method so
1.185 + //we must not return anything to prevent cycling
1.186 + return null;
1.187 + }
1.188 + //get the encoding from the FEQ excluding this FEQ implementation
1.189 + //so the proxy charset can default to appropriate encoding.
1.190 + //@todo Caching?
1.191 + callingFEQ.set(true);
1.192 + try {
1.193 + Charset charset = FileEncodingQuery.getEncoding(file);
1.194 + return new ProxyCharset(charset);
1.195 + } finally {
1.196 + callingFEQ.set(false);
1.197 + }
1.198 + }
1.199 +
1.200 + private class ProxyCharset extends Charset {
1.201 +
1.202 + public ProxyCharset (Charset charset) {
1.203 + super (charset.name(), new String[0]); //NOI18N
1.204 + }
1.205 +
1.206 + public boolean contains(Charset c) {
1.207 + return false;
1.208 + }
1.209 +
1.210 + public CharsetDecoder newDecoder() {
1.211 + return new PythonDecoder(this);
1.212 + }
1.213 +
1.214 + public CharsetEncoder newEncoder() {
1.215 + return new PythonEncoder(this);
1.216 + }
1.217 + }
1.218 +
1.219 +
1.220 + private class PythonEncoder extends CharsetEncoder {
1.221 + private CharBuffer buffer = CharBuffer.allocate(4*1024);
1.222 + private CharBuffer remainder;
1.223 + private CharsetEncoder encoder;
1.224 +
1.225 + public PythonEncoder(Charset cs) {
1.226 + super(cs, 1,2);
1.227 + }
1.228 +
1.229 +
1.230 + @Override
1.231 + protected CoderResult encodeLoop(CharBuffer in, ByteBuffer out) {
1.232 + if (buffer == null) {
1.233 + assert encoder != null;
1.234 + if (remainder!=null) {
1.235 + CoderResult result = encoder.encode(remainder,out,false);
1.236 + if (!remainder.hasRemaining()) {
1.237 + remainder = null;
1.238 + }
1.239 + }
1.240 + CoderResult result = encoder.encode(in, out, false);
1.241 + return result;
1.242 + }
1.243 + if (buffer.remaining() == 0 || (buffer.position() > 0 && in.limit() == 0)) {
1.244 + return handleHead (in,out);
1.245 + }
1.246 + else if (buffer.remaining() < in.remaining()) {
1.247 + int limit = in.limit();
1.248 + in.limit(in.position()+buffer.remaining());
1.249 + buffer.put(in);
1.250 + in.limit(limit);
1.251 + return handleHead (in, out);
1.252 + }
1.253 + else {
1.254 + buffer.put(in);
1.255 + return CoderResult.UNDERFLOW;
1.256 + }
1.257 + }
1.258 +
1.259 + private CoderResult handleHead (CharBuffer in, ByteBuffer out) {
1.260 + String encoding = null;
1.261 + try {
1.262 + encoding = getEncoding ();
1.263 + } catch (IOException ioe) {
1.264 + Exceptions.printStackTrace(ioe);
1.265 + }
1.266 + if (encoding == null) {
1.267 + buffer = null;
1.268 + throwUnknownEncoding();
1.269 + return null;
1.270 + }
1.271 + else {
1.272 + Charset c;
1.273 + try {
1.274 + c = Charset.forName(encoding);
1.275 + } catch (UnsupportedCharsetException | IllegalCharsetNameException e) {
1.276 + buffer = null;
1.277 + throwUnknownEncoding();
1.278 + return null;
1.279 + }
1.280 + encoder = c.newEncoder();
1.281 + return flushHead(in, out);
1.282 + }
1.283 + }
1.284 +
1.285 + private CoderResult flushHead (CharBuffer in , ByteBuffer out) {
1.286 + buffer.flip();
1.287 + CoderResult r = encoder.encode(buffer,out, in==null);
1.288 + if (r.isOverflow()) {
1.289 + remainder = buffer;
1.290 + buffer = null;
1.291 + return r;
1.292 + }
1.293 + else {
1.294 + buffer = null;
1.295 + if (in == null) {
1.296 + return r;
1.297 + }
1.298 + return encoder.encode(in, out, false);
1.299 + }
1.300 + }
1.301 +
1.302 + private String getEncoding () throws IOException {
1.303 + String text = buffer.asReadOnlyBuffer().flip().toString();
1.304 + InputStream in = new ByteArrayInputStream(text.getBytes());
1.305 + try {
1.306 + return getPythonFileEncoding(in);
1.307 + } finally {
1.308 + in.close();
1.309 + }
1.310 + }
1.311 +
1.312 + @Override
1.313 + protected CoderResult implFlush(ByteBuffer out) {
1.314 + CoderResult res;
1.315 + if (buffer != null) {
1.316 + res = handleHead(null, out);
1.317 + return res;
1.318 + }
1.319 + else if (remainder != null) {
1.320 + encoder.encode(remainder, out, true);
1.321 + }
1.322 + else {
1.323 + CharBuffer empty = (CharBuffer) CharBuffer.allocate(0).flip();
1.324 + encoder.encode(empty, out, true);
1.325 + }
1.326 + res = encoder.flush(out);
1.327 + return res;
1.328 + }
1.329 +
1.330 + @Override
1.331 + protected void implReset() {
1.332 + if (encoder != null) {
1.333 + encoder.reset();
1.334 + }
1.335 + }
1.336 + }
1.337 +
1.338 +
1.339 + private class PythonDecoder extends CharsetDecoder {
1.340 +
1.341 + private ByteBuffer buffer = ByteBuffer.allocate(4*1024);
1.342 + private ByteBuffer remainder;
1.343 + private CharsetDecoder decoder;
1.344 +
1.345 + public PythonDecoder(Charset cs) {
1.346 + super (cs,1,2);
1.347 + }
1.348 +
1.349 + protected CoderResult decodeLoop(ByteBuffer in, CharBuffer out) {
1.350 + if (buffer == null) {
1.351 + assert decoder != null;
1.352 + if (remainder!=null) {
1.353 + ByteBuffer tmp = ByteBuffer.allocate(remainder.remaining() + in.remaining());
1.354 + tmp.put(remainder);
1.355 + tmp.put(in);
1.356 + tmp.flip();
1.357 + CoderResult result = decoder.decode(tmp,out,false);
1.358 + if (tmp.hasRemaining()) {
1.359 + remainder = tmp;
1.360 + }
1.361 + else {
1.362 + remainder = null;
1.363 + }
1.364 + return result;
1.365 + }
1.366 + else {
1.367 + return decoder.decode(in, out, false);
1.368 + }
1.369 + }
1.370 + if (buffer.remaining() == 0) {
1.371 + return handleHead (in,out);
1.372 + }
1.373 + else if (buffer.remaining() < in.remaining()) {
1.374 + int limit = in.limit();
1.375 + in.limit(in.position()+buffer.remaining());
1.376 + buffer.put(in);
1.377 + in.limit(limit);
1.378 + return handleHead (in, out);
1.379 + }
1.380 + else {
1.381 + buffer.put(in);
1.382 + return CoderResult.UNDERFLOW;
1.383 + }
1.384 + }
1.385 +
1.386 + private CoderResult handleHead (ByteBuffer in, CharBuffer out) {
1.387 + String encoding = null;
1.388 + try {
1.389 + encoding = getEncoding ();
1.390 + } catch (IOException ioe) {
1.391 + Exceptions.printStackTrace(ioe);
1.392 + }
1.393 + if (encoding == null) {
1.394 + buffer = null;
1.395 + throwUnknownEncoding();
1.396 + return null;
1.397 + }
1.398 + else {
1.399 + Charset c;
1.400 + try {
1.401 + //c = cache(Charset.forName(encoding));
1.402 + c = Charset.forName(encoding);
1.403 + } catch (UnsupportedCharsetException e) {
1.404 + buffer = null;
1.405 + throwUnknownEncoding();
1.406 + return null;
1.407 + } catch (IllegalCharsetNameException e) {
1.408 + buffer = null;
1.409 + throwUnknownEncoding();
1.410 + return null;
1.411 + }
1.412 + decoder = c.newDecoder();
1.413 + return flushHead(in, out);
1.414 + }
1.415 + }
1.416 +
1.417 + private CoderResult flushHead (ByteBuffer in , CharBuffer out) {
1.418 + buffer.flip();
1.419 + CoderResult r = decoder.decode(buffer,out, in==null);
1.420 + if (r.isOverflow()) {
1.421 + remainder = buffer;
1.422 + buffer = null;
1.423 + return r;
1.424 + }
1.425 + else {
1.426 + buffer = null;
1.427 + if (in == null) {
1.428 + return r;
1.429 + }
1.430 + return decoder.decode(in, out, false);
1.431 + }
1.432 + }
1.433 +
1.434 + private String getEncoding () throws IOException {
1.435 + byte[] arr = buffer.array();
1.436 + ByteArrayInputStream in = new ByteArrayInputStream (arr);
1.437 + try {
1.438 + return getPythonFileEncoding(in);
1.439 + }
1.440 + finally {
1.441 + in.close();
1.442 + }
1.443 + }
1.444 +
1.445 + @Override
1.446 + protected CoderResult implFlush(CharBuffer out) {
1.447 + CoderResult res;
1.448 + if (buffer != null) {
1.449 + res = handleHead(null, out);
1.450 + return res;
1.451 + }
1.452 + else if (remainder != null) {
1.453 + decoder.decode(remainder, out, true);
1.454 + }
1.455 + else {
1.456 + ByteBuffer empty = (ByteBuffer) ByteBuffer.allocate(0).flip();
1.457 + decoder.decode(empty, out, true);
1.458 + }
1.459 + res = decoder.flush(out);
1.460 + return res;
1.461 + }
1.462 +
1.463 + @Override
1.464 + protected void implReset() {
1.465 + if (decoder != null) {
1.466 + decoder.reset();
1.467 + }
1.468 + }
1.469 + }
1.470 +}
2.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
2.2 +++ b/python.project/src/org/netbeans/modules/python/project/PythonProjectTemplateAttributesProvider.java Fri Jul 17 07:47:00 2015 +0200
2.3 @@ -0,0 +1,75 @@
2.4 +/*
2.5 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
2.6 + *
2.7 + * Copyright 2015 Oracle and/or its affiliates. All rights reserved.
2.8 + *
2.9 + * Oracle and Java are registered trademarks of Oracle and/or its affiliates.
2.10 + * Other names may be trademarks of their respective owners.
2.11 + *
2.12 + * The contents of this file are subject to the terms of either the GNU
2.13 + * General Public License Version 2 only ("GPL") or the Common
2.14 + * Development and Distribution License("CDDL") (collectively, the
2.15 + * "License"). You may not use this file except in compliance with the
2.16 + * License. You can obtain a copy of the License at
2.17 + * http://www.netbeans.org/cddl-gplv2.html
2.18 + * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
2.19 + * specific language governing permissions and limitations under the
2.20 + * License. When distributing the software, include this License Header
2.21 + * Notice in each file and include the License file at
2.22 + * nbbuild/licenses/CDDL-GPL-2-CP. Oracle designates this
2.23 + * particular file as subject to the "Classpath" exception as provided
2.24 + * by Oracle in the GPL Version 2 section of the License file that
2.25 + * accompanied this code. If applicable, add the following below the
2.26 + * License Header, with the fields enclosed by brackets [] replaced by
2.27 + * your own identifying information:
2.28 + * "Portions Copyrighted [year] [name of copyright owner]"
2.29 + *
2.30 + * If you wish your version of this file to be governed by only the CDDL
2.31 + * or only the GPL Version 2, indicate your decision by adding
2.32 + * "[Contributor] elects to include this software in this distribution
2.33 + * under the [CDDL or GPL Version 2] license." If you do not indicate a
2.34 + * single choice of license, a recipient has the option to distribute
2.35 + * your version of this file under either the CDDL, the GPL Version 2 or
2.36 + * to extend the choice of license to its licensees as provided above.
2.37 + * However, if you add GPL Version 2 code and therefore, elected the GPL
2.38 + * Version 2 license, then the option applies only if the new code is
2.39 + * made subject to such option by the copyright holder.
2.40 + *
2.41 + * Contributor(s):
2.42 + *
2.43 + * Portions Copyrighted 2015 Sun Microsystems, Inc.
2.44 + */
2.45 +package org.netbeans.modules.python.project;
2.46 +
2.47 +import java.nio.charset.Charset;
2.48 +import java.util.HashMap;
2.49 +import java.util.Map;
2.50 +import org.netbeans.modules.python.project.ui.customizer.PythonProjectProperties;
2.51 +import org.netbeans.spi.project.support.ant.PropertyEvaluator;
2.52 +import org.openide.filesystems.FileObject;
2.53 +import org.openide.loaders.CreateFromTemplateAttributesProvider;
2.54 +import org.openide.loaders.DataFolder;
2.55 +import org.openide.loaders.DataObject;
2.56 +
2.57 +/**
2.58 + *
2.59 + * @author Vincent van der Leun
2.60 + */
2.61 +
2.62 +public class PythonProjectTemplateAttributesProvider implements CreateFromTemplateAttributesProvider {
2.63 + private final PropertyEvaluator eval;
2.64 +
2.65 + public PythonProjectTemplateAttributesProvider(final PropertyEvaluator eval) {
2.66 + this.eval = eval;
2.67 + }
2.68 +
2.69 + @Override
2.70 + public Map<String, ?> attributesFor(DataObject template, DataFolder target, String name) {
2.71 + Map<String,Object> result = new HashMap<>();
2.72 + String encoding = eval.getProperty(PythonProjectProperties.SOURCE_ENCODING);
2.73 + if (encoding != null) {
2.74 + result.put("encoding", encoding); //NOI18N
2.75 + }
2.76 + return result;
2.77 + }
2.78 +}