Wrong charset used in creating search terms when server supports UTF8

providers/angus-mail/src/test/java/org/eclipse/angus/mail/imap/IMAPSearchTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2009, 2023 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2009, 2024 Oracle and/or its affiliates. All rights reserved.
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Public License v. 2.0, which is available at
@@ -22,12 +22,18 @@
 import jakarta.mail.Store;
 import jakarta.mail.search.SearchException;
 import jakarta.mail.search.SubjectTerm;
+import org.eclipse.angus.mail.imap.protocol.IMAPProtocol;
+import org.eclipse.angus.mail.imap.protocol.SearchSequence;
 import org.eclipse.angus.mail.test.TestServer;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.Timeout;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
 import java.util.Properties;
 
 import static org.junit.Assert.fail;
@@ -95,13 +101,17 @@ public void search(String line) throws IOException {
      * term includes a non-ASCII character.
      * (see RFC 6855, section 3, last paragraph)
      */
+    //TODO: Fix this TestServer/ProtocolHandler so this test passes.
+    //TODO: Fix the test.
     @Test
     public void testUtf8Search() {
+        final String find = "\u2019\u7cfb\u7edf";
         TestServer server = null;
         try {
             server = new TestServer(new IMAPUtf8Handler() {
                 @Override
                 public void search(String line) throws IOException {
+                    System.out.println(line);
                     if (line.contains("CHARSET"))
                         bad("CHARSET not supported");
                     else
@@ -115,14 +125,15 @@ public void search(String line) throws IOException {
             properties.setProperty("mail.imap.port", String.valueOf(server.getPort()));
             final Session session = Session.getInstance(properties);
             //session.setDebug(true);
-
+            
             final Store store = session.getStore("imap");
             Folder folder = null;
             try {
                 store.connect("test", "test");
                 folder = store.getFolder("INBOX");
                 folder.open(Folder.READ_ONLY);
-                Message[] msgs = folder.search(new SubjectTerm("\u2019"));
+                SubjectTerm term = new SubjectTerm(find);
+                Message[] msgs = folder.search(term);
             } catch (Exception ex) {
                 System.out.println(ex);
                 //ex.printStackTrace();
@@ -141,6 +152,28 @@ public void search(String line) throws IOException {
             }
         }
     }
+
+    //@Test
+    @org.junit.Ignore
+    public void testUtf8SubjectLiteral() throws Exception {
+        final String find = "\u2019\u7cfb\u7edf";
+        SubjectTerm term = new SubjectTerm(find);
+        InputStream in = new ByteArrayInputStream(find.getBytes("UTF-8"));
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        PrintStream out = new PrintStream(baos, true, "UTF-8");
+        Properties props = new Properties();
+
+        IMAPProtocol p = new IMAPProtocol(in, out, props, false) {
+            public boolean supportsUtf8() {
+                return true;
+            }
+        };
+
+        SearchSequence ss = new SearchSequence(p);
+        ss.generateSequence(term, "UTF-8").write(p);
+
+        System.out.println(baos.toString("UTF-8"));
+    }
 
     /**
      * An IMAPHandler that enables UTF-8 support.

providers/imap/src/main/java/org/eclipse/angus/mail/imap/protocol/IMAPProtocol.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2023 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024 Oracle and/or its affiliates. All rights reserved.
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Public License v. 2.0, which is available at
@@ -452,6 +452,7 @@ public Map<String, String> getCapabilities() {
      *
      * @since JavaMail 1.6.0
      */
+    @Override
     public boolean supportsUtf8() {
         return utf8;
     }
@@ -2496,9 +2497,11 @@ private int[] search(String msgSequence, SearchTerm term)
         // Check if the search "text" terms contain only ASCII chars,
         // or if utf8 support has been enabled (in which case CHARSET
         // is not allowed; see RFC 6855, section 3, last paragraph)
-        if (supportsUtf8() || SearchSequence.isAscii(term)) {
+        String charset = supportsUtf8() ? MimeUtility.mimeCharset(
+                StandardCharsets.UTF_8.name()) : null;
+        if (charset != null || SearchSequence.isAscii(term)) {
             try {
-                return issueSearch(msgSequence, term, null);
+                return issueSearch(msgSequence, term, charset);
             } catch (IOException ioex) { /* will not happen */ }
         }
 
@@ -2550,14 +2553,17 @@ private int[] issueSearch(String msgSequence, SearchTerm term,
 
         // Generate a search-sequence with the given charset
         Argument args = getSearchSequence().generateSequence(term,
-                charset == null ? null :
+                charset == null ? (String) null :
                         MimeUtility.javaCharset(charset)
         );
         args.writeAtom(msgSequence);
 
+        //A charset of null in this specific case means term is ASCII only.
+        //If term is ASCII only or if utf8 support has been enabled
+        //(in which case CHARSET is not allowed; see RFC 6855, section 3,
+        //last paragraph)
         Response[] r;
-
-        if (charset == null) // text is all US-ASCII
+        if (charset == null || supportsUtf8())
             r = command("SEARCH", args);
         else
             r = command("SEARCH CHARSET " + charset, args);
@@ -2873,7 +2879,7 @@ public void setQuota(Quota quota) throws ProtocolException {
 	List<Quota> v = new ArrayList<Quota>();
 
 	// Grab all QUOTA responses
-	if (response.isOK()) { // command succesful 
+	if (response.isOK()) { // command succesful
 	    for (int i = 0, len = r.length; i < len; i++) {
 		if (!(r[i] instanceof IMAPResponse))
 		    continue;

providers/imap/src/main/java/org/eclipse/angus/mail/imap/protocol/SearchSequence.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2023 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024 Oracle and/or its affiliates. All rights reserved.
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Public License v. 2.0, which is available at
@@ -40,11 +40,13 @@
 import jakarta.mail.search.StringTerm;
 import jakarta.mail.search.SubjectTerm;
 import org.eclipse.angus.mail.iap.Argument;
+import org.eclipse.angus.mail.iap.Literal;
 import org.eclipse.angus.mail.imap.ModifiedSinceTerm;
 import org.eclipse.angus.mail.imap.OlderTerm;
 import org.eclipse.angus.mail.imap.YoungerTerm;
 
 import java.io.IOException;
+import java.io.OutputStream;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.GregorianCalendar;
@@ -62,7 +64,7 @@
  */
 public class SearchSequence {
 
-    private IMAPProtocol protocol;    // for hasCapability checks; may be null
+    private final IMAPProtocol protocol;    // for hasCapability checks; may be null
 
     /**
      * Create a SearchSequence for this IMAPProtocol.
@@ -79,6 +81,7 @@ public SearchSequence(IMAPProtocol p) {
      */
     @Deprecated
     public SearchSequence() {
+        protocol = null; //TODO Deprecate for remove???
     }
 
     /**
@@ -283,7 +286,12 @@ protected Argument header(HeaderTerm term, String charset)
         Argument result = new Argument();
         result.writeAtom("HEADER");
         result.writeString(term.getHeaderName());
-        result.writeString(term.getPattern(), charset);
+        String pattern = term.getPattern();
+        if (protocol != null && protocol.supportsUtf8() && !isAscii(term)) {
+            result.writeBytes(new Utf8Literal(pattern, charset));
+        } else {
+            result.writeString(pattern, charset);
+        }
         return result;
     }
 
@@ -335,7 +343,11 @@ protected Argument from(String address, String charset)
             throws SearchException, IOException {
         Argument result = new Argument();
         result.writeAtom("FROM");
-        result.writeString(address, charset);
+        if (protocol != null && protocol.supportsUtf8() && !isAscii(address)) {
+            result.writeBytes(new Utf8Literal(address, charset));
+        } else {
+            result.writeString(address, charset);
+        }
         return result;
     }
 
@@ -353,7 +365,12 @@ else if (type == Message.RecipientType.BCC)
         else
             throw new SearchException("Illegal Recipient type");
 
-        result.writeString(address, charset);
+
+        if (protocol != null && protocol.supportsUtf8() && !isAscii(address)) {
+            result.writeBytes(new Utf8Literal(address, charset));
+        } else {
+            result.writeString(address, charset);
+        }
         return result;
     }
 
@@ -362,7 +379,12 @@ protected Argument subject(SubjectTerm term, String charset)
         Argument result = new Argument();
 
         result.writeAtom("SUBJECT");
-        result.writeString(term.getPattern(), charset);
+        String pattern = term.getPattern();
+        if (protocol != null && protocol.supportsUtf8() && !isAscii(term)) {
+            result.writeBytes(new Utf8Literal(pattern, charset));
+        } else {
+            result.writeString(pattern, charset);
+        }
         return result;
     }
 
@@ -371,7 +393,12 @@ protected Argument body(BodyTerm term, String charset)
         Argument result = new Argument();
 
         result.writeAtom("BODY");
-        result.writeString(term.getPattern(), charset);
+        String pattern = term.getPattern();
+        if (protocol != null && protocol.supportsUtf8() && !isAscii(term)) {
+            result.writeBytes(new Utf8Literal(pattern, charset));
+        } else {
+            result.writeString(pattern, charset);
+        }
         return result;
     }
 
@@ -410,7 +437,7 @@ protected Argument size(SizeTerm term)
      *
      * Note that this format does not contain the TimeZone
      */
-    private static String[] monthTable = {
+    private static final String[] monthTable = {
             "Jan", "Feb", "Mar", "Apr", "May", "Jun",
             "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
     };
@@ -543,4 +570,22 @@ protected Argument modifiedSince(ModifiedSinceTerm term)
         result.writeNumber(term.getModSeq());
         return result;
     }
+
+    private static final class Utf8Literal implements Literal {
+        private final byte[] bytes;
+
+        Utf8Literal(String data, String charset) throws IOException {
+            this.bytes = data.getBytes(charset == null ? "UTF-8" : charset);
+        }
+
+        @Override
+        public int size() {
+            return bytes.length;
+        }
+
+        @Override
+        public void writeTo(OutputStream os) throws IOException {
+            os.write(this.bytes);
+        }
+    }
 }