Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 55c55835 authored by Matt Garnes's avatar Matt Garnes Committed by Roman Birg
Browse files

Detect text encoding with juniversalchardet. 

- When opening files in the built in editor for display, detect the
  encoding with juniversalchardet, so that the correct encoding will be
  used.
- Use byte buffers to back ShellConsole instead of StringBuffers

Change-Id: I85fa567ef589a82f1c8604f1f215647376c31c9a
parent e9303a38

Android.mk

+6 −0
Original line number Diff line number Diff line
@@ -23,11 +23,17 @@ LOCAL_SRC_FILES += $(call all-java-files-under, libs/color-picker-view/src) 


LOCAL_STATIC_JAVA_LIBRARIES += libtruezip

LOCAL_STATIC_JAVA_LIBRARIES += android-support-v4

LOCAL_STATIC_JAVA_LIBRARIES += juniversalchardet



LOCAL_PACKAGE_NAME := CMFileManager

LOCAL_CERTIFICATE := platform

LOCAL_PROGUARD_FLAG_FILES := proguard.flags



include $(BUILD_PACKAGE)

include $(CLEAR_VARS)



LOCAL_PREBUILT_STATIC_JAVA_LIBRARIES := \

    juniversalchardet:libs/juniversalchardet/juniversalchardet-1.0.3.jar



include $(BUILD_MULTI_PREBUILT)

include $(call all-makefiles-under,$(LOCAL_PATH))

libs/juniversalchardet/LICENSE

0 → 100644
+470 −0

File added.

Preview size limit exceeded, changes collapsed.

libs/juniversalchardet/juniversalchardet-1.0.3.jar

0 → 100644
+202 KiB

File added.

No diff preview for this file type.

View file

src/com/cyanogenmod/filemanager/activities/EditorActivity.java

+31 −4
Original line number Diff line number Diff line
@@ -68,7 +68,12 @@ import com.cyanogenmod.filemanager.ash.SyntaxHighlightFactory; 
import com.cyanogenmod.filemanager.ash.SyntaxHighlightProcessor;

import com.cyanogenmod.filemanager.commands.AsyncResultListener;

import com.cyanogenmod.filemanager.commands.WriteExecutable;

import com.cyanogenmod.filemanager.commands.shell.InvalidCommandDefinitionException;

import com.cyanogenmod.filemanager.console.Console;

import com.cyanogenmod.filemanager.console.ConsoleAllocException;

import com.cyanogenmod.filemanager.console.ConsoleBuilder;

import com.cyanogenmod.filemanager.console.InsufficientPermissionsException;

import com.cyanogenmod.filemanager.console.java.JavaConsole;

import com.cyanogenmod.filemanager.model.FileSystemObject;

import com.cyanogenmod.filemanager.preferences.FileManagerSettings;

import com.cyanogenmod.filemanager.preferences.Preferences;
@@ -85,11 +90,13 @@ import com.cyanogenmod.filemanager.util.FileHelper; 
import com.cyanogenmod.filemanager.util.MediaHelper;

import com.cyanogenmod.filemanager.util.ResourcesHelper;

import com.cyanogenmod.filemanager.util.StringHelper;

import org.mozilla.universalchardet.UniversalDetector;



import java.io.BufferedReader;

import java.io.ByteArrayInputStream;

import java.io.ByteArrayOutputStream;

import java.io.File;

import java.io.IOException;

import java.io.OutputStream;

import java.io.StringReader;

import java.util.ArrayList;
@@ -273,12 +280,19 @@ public class EditorActivity extends Activity implements TextWatcher { 
        long mSize;

        FileSystemObject mReadFso;

        OnProgressListener mListener;

        boolean mDetectEncoding = false;

        UniversalDetector mDetector;

        String mDetectedEncoding;



        /**

         * Constructor of <code>AsyncReader</code>. For enclosing access.

         */

        public AsyncReader() {

        public AsyncReader(boolean detectEncoding) {

            super();

            mDetectEncoding = detectEncoding;

            if (mDetectEncoding) {

                mDetector = new UniversalDetector(null);

            }

        }



        /**
@@ -298,6 +312,9 @@ public class EditorActivity extends Activity implements TextWatcher { 
            if (!cancelled && StringHelper.isBinaryData(mByteBuffer.toByteArray())) {

                EditorActivity.this.mBinary = true;

                EditorActivity.this.mReadOnly = true;

            } else if (mDetector != null) {

                mDetector.dataEnd();

                mDetectedEncoding = mDetector.getDetectedCharset();

            }

        }
@@ -319,6 +336,9 @@ public class EditorActivity extends Activity implements TextWatcher { 
            try {

                if (result == null) return;

                byte[] partial = (byte[]) result;

                if (mDetectEncoding) {

                    mDetector.handleData(partial, 0, partial.length);

                }

                this.mByteBuffer.write(partial, 0, partial.length);

                this.mSize += partial.length;

                if (this.mListener != null && this.mReadFso != null) {
@@ -1087,7 +1107,7 @@ public class EditorActivity extends Activity implements TextWatcher { 
                try {

                    while (true) {

                        // Configure the reader

                        this.mReader = new AsyncReader();

                        this.mReader = new AsyncReader(true);

                        this.mReader.mReadFso = fso;

                        this.mReader.mListener = new OnProgressListener() {

                            @Override
@@ -1098,7 +1118,8 @@ public class EditorActivity extends Activity implements TextWatcher { 
                        };



                        // Execute the command (read the file)

                        CommandHelper.read(activity, fso.getFullPath(), this.mReader, null);

                        CommandHelper.read(activity, fso.getFullPath(), this.mReader,

                                           null);



                        // Wait for

                        synchronized (this.mReader.mSync) {
@@ -1132,7 +1153,13 @@ public class EditorActivity extends Activity implements TextWatcher { 
                        }

                        Log.i(TAG, "Bytes read: " + data.length()); //$NON-NLS-1$

                    } else {

                        final String data = new String(this.mReader.mByteBuffer.toByteArray());

                        String data;

                        if (this.mReader.mDetectedEncoding != null) {

                            data = new String(this.mReader.mByteBuffer.toByteArray(),

                                              this.mReader.mDetectedEncoding);

                        } else {

                            data = new String(this.mReader.mByteBuffer.toByteArray());

                        }

                        this.mReader.mBuffer = new SpannableStringBuilder(data);

                        Log.i(TAG, "Bytes read: " + data.getBytes().length); //$NON-NLS-1$

                    }

src/com/cyanogenmod/filemanager/commands/shell/AsyncResultProgram.java

+8 −9
Original line number Diff line number Diff line
@@ -47,7 +47,7 @@ public abstract class AsyncResultProgram 
    /**

     * @hide

     */

    final List<String> mPartialData;

    final List<byte[]> mPartialData;

    /**

     * @hide

     */
@@ -97,7 +97,7 @@ public abstract class AsyncResultProgram 
        if (mAsyncResultListener instanceof ConcurrentAsyncResultListener) {

            ((ConcurrentAsyncResultListener) mAsyncResultListener).onRegister();

        }

        this.mPartialData = Collections.synchronizedList(new ArrayList<String>());

        this.mPartialData = Collections.synchronizedList(new ArrayList<byte[]>());

        this.mPartialDataType = Collections.synchronizedList(new ArrayList<Byte>());

        this.mTempBuffer = new StringBuffer();

        this.mOnCancelListener = null;
@@ -169,12 +169,12 @@ public abstract class AsyncResultProgram 
    /**

     * Method that parse the result of a program invocation.

     *

     * @param partialIn A partial standard input buffer (incremental buffer)

     * @param input A partial standard input buffer (incremental buffer)

     * @hide

     */

    public final void onRequestParsePartialResult(String partialIn) {

    public final void onRequestParsePartialResult(byte[] input) {

        String partialIn = new String(input);

        synchronized (this.mSync) {

            String data = partialIn;

            String rest = ""; //$NON-NLS-1$

            if (parseOnlyCompleteLines()) {

                int pos = partialIn.lastIndexOf(FileHelper.NEWLINE);
@@ -185,12 +185,11 @@ public abstract class AsyncResultProgram 
                }



                //Retrieve the data

                data = this.mTempBuffer.append(partialIn.substring(0, pos + 1)).toString();

                rest = partialIn.substring(pos + 1);

            }



            this.mPartialDataType.add(STDIN);

            this.mPartialData.add(data);

            this.mPartialData.add(input);

            this.mTempBuffer = new StringBuffer(rest);

            this.mSync.notify();

        }
@@ -220,7 +219,7 @@ public abstract class AsyncResultProgram 
            }



            this.mPartialDataType.add(STDERR);

            this.mPartialData.add(data);

            this.mPartialData.add(data.getBytes());

            this.mTempBuffer = new StringBuffer(rest);

            this.mSync.notify();

        }
@@ -381,7 +380,7 @@ public abstract class AsyncResultProgram 
                       AsyncResultProgram.this.mSync.wait();

                       while (AsyncResultProgram.this.mPartialData.size() > 0) {

                           Byte type = AsyncResultProgram.this.mPartialDataType.remove(0);

                           String data = AsyncResultProgram.this.mPartialData.remove(0);

                           byte[] data = AsyncResultProgram.this.mPartialData.remove(0);

                           try {

                               if (type.compareTo(STDIN) == 0) {

                                   AsyncResultProgram.this.onParsePartialResult(data);