Merge pull request #833 from symbol/fix/metadata-utf8-encoding-fix

fix: metadata value non-ascii utf8 encoding issue fixed

Author: yilmazbahadir

e2e/infrastructure/MetadataHttp.spec.ts

@@ -18,6 +18,7 @@ import { deepEqual } from 'assert';
 import { expect } from 'chai';
 import { firstValueFrom } from 'rxjs';
 import { take, toArray } from 'rxjs/operators';
+import { Convert } from '../..';
 import { MetadataPaginationStreamer, MetadataRepository, Order } from '../../src/infrastructure';
 import { Metadata, MetadataType, UInt64 } from '../../src/model';
 import { Account, Address } from '../../src/model/account';
@@ -115,7 +116,7 @@ describe('MetadataHttp', () => {
                 account.address,
                 UInt64.fromUint(6),
                 23,
-                `Test account meta value`,
+                Convert.utf8ToUint8(`Test account meta value`),
                 networkType,
                 helper.maxFee,
             );
@@ -140,7 +141,7 @@ describe('MetadataHttp', () => {
                 UInt64.fromUint(6),
                 mosaicId,
                 22,
-                `Test mosaic meta value`,
+                Convert.utf8ToUint8(`Test mosaic meta value`),
                 networkType,
                 helper.maxFee,
             );
@@ -165,7 +166,7 @@ describe('MetadataHttp', () => {
                 UInt64.fromUint(6),
                 namespaceId,
                 25,
-                `Test namespace meta value`,
+                Convert.utf8ToUint8(`Test namespace meta value`),
                 networkType,
                 helper.maxFee,
             );

e2e/infrastructure/TransactionHttp.spec.ts

@@ -196,7 +196,7 @@ describe('TransactionHttp', () => {
                 account.address,
                 UInt64.fromUint(5),
                 10,
-                Convert.uint8ToUtf8(new Uint8Array(10)),
+                new Uint8Array(10),
                 networkType,
                 helper.maxFee,
             );
@@ -229,7 +229,7 @@ describe('TransactionHttp', () => {
                 UInt64.fromUint(5),
                 mosaicId,
                 10,
-                Convert.uint8ToUtf8(new Uint8Array(10)),
+                new Uint8Array(10),
                 networkType,
                 helper.maxFee,
             );
@@ -323,7 +323,7 @@ describe('TransactionHttp', () => {
                 UInt64.fromUint(5),
                 addressAlias,
                 10,
-                Convert.uint8ToUtf8(new Uint8Array(10)),
+                new Uint8Array(10),
                 networkType,
                 helper.maxFee,
             );

e2e/infrastructure/UnresolvedMapping.spec.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 import { expect } from 'chai';
-import { Convert } from '../../src/core/format';
 import { Account } from '../../src/model/account';
 import { PlainMessage } from '../../src/model/message/PlainMessage';
 import { MosaicFlags, MosaicId, MosaicNonce } from '../../src/model/mosaic';
@@ -165,7 +164,7 @@ describe('Unresolved Mapping', () => {
                 UInt64.fromUint(5),
                 namespaceIdMosaic,
                 10,
-                Convert.uint8ToUtf8(new Uint8Array(10)),
+                new Uint8Array(10),
                 networkType,
                 helper.maxFee,
             );

e2e/service/MetadataTransactionService.spec.ts

@@ -95,7 +95,7 @@ describe('MetadataTransactionService', () => {
                 key,
                 mosaicId,
                 newValue.length,
-                newValue,
+                Convert.utf8ToUint8(newValue),
                 networkType,
                 helper.maxFee,
             );
@@ -121,7 +121,7 @@ describe('MetadataTransactionService', () => {
                 key,
                 namespaceId,
                 newValue.length,
-                newValue,
+                Convert.utf8ToUint8(newValue),
                 networkType,
             );
 

e2e/service/TransactionService.spec.ts

@@ -16,7 +16,6 @@
 
 import { assert, expect } from 'chai';
 import { firstValueFrom } from 'rxjs';
-import { Convert } from '../../src/core/format';
 import { TransactionRepository } from '../../src/infrastructure/TransactionRepository';
 import { Account, Address } from '../../src/model/account';
 import { PlainMessage } from '../../src/model/message/PlainMessage';
@@ -136,7 +135,7 @@ describe('TransactionService', () => {
             UInt64.fromUint(5),
             mosaicAlias,
             10,
-            Convert.uint8ToUtf8(new Uint8Array(10)),
+            new Uint8Array(10),
             networkType,
             helper.maxFee,
         );

src/core/format/Convert.ts

@@ -26,7 +26,7 @@ export class Convert {
     public static toByte = (char1: string, char2: string): number => {
         const byte = utilities.tryParseByte(char1, char2);
         if (undefined === byte) {
-            throw Error(`unrecognized hex char`);
+            throw Error(`unrecognized hex char, char1:${char1}, char2:${char2}`);
         }
         return byte;
     };
@@ -148,33 +148,6 @@ export class Convert {
         return input & 0xff;
     };
 
-    /**
-     * Converts a raw javascript string into a string of single byte characters using utf8 encoding.
-     * This makes it easier to perform other encoding operations on the string.
-     * @param {string} input - A raw string
-     * @return {string} - UTF-8 string
-     */
-    public static rstr2utf8 = (input: string): string => {
-        let output = '';
-
-        for (let n = 0; n < input.length; n++) {
-            const c = input.charCodeAt(n);
-
-            if (128 > c) {
-                output += String.fromCharCode(c);
-            } else if (127 < c && 2048 > c) {
-                output += String.fromCharCode((c >> 6) | 192);
-                output += String.fromCharCode((c & 63) | 128);
-            } else {
-                output += String.fromCharCode((c >> 12) | 224);
-                output += String.fromCharCode(((c >> 6) & 63) | 128);
-                output += String.fromCharCode((c & 63) | 128);
-            }
-        }
-
-        return output;
-    };
-
     /**
      * Convert UTF-8 to hex
      * @param {string} input - An UTF-8 string
@@ -190,7 +163,7 @@ export class Convert {
      * @return {Uint8Array}
      */
     public static utf8ToUint8 = (input: string): Uint8Array => {
-        const hex = Convert.utf8ToHex(Convert.rstr2utf8(input));
+        const hex = Convert.utf8ToHex(input);
         return Convert.hexToUint8(hex);
     };
 

src/infrastructure/transaction/CreateTransactionFromDTO.ts

@@ -13,7 +13,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Convert as convert } from '../../core/format';
 import { UnresolvedMapping } from '../../core/utils';
 import { MessageFactory, MosaicSupplyRevocationTransaction, UInt64 } from '../../model';
 import { Address, PublicAccount } from '../../model/account';
@@ -393,7 +392,7 @@ const CreateStandaloneTransactionFromDTO = (transactionDTO, transactionInfo): Tr
                 extractRecipient(transactionDTO.targetAddress),
                 UInt64.fromHex(transactionDTO.scopedMetadataKey),
                 transactionDTO.valueSizeDelta,
-                convert.decodeHex(transactionDTO.value),
+                transactionDTO.value,
                 signature,
                 signer,
                 transactionInfo,
@@ -408,7 +407,7 @@ const CreateStandaloneTransactionFromDTO = (transactionDTO, transactionInfo): Tr
                 UInt64.fromHex(transactionDTO.scopedMetadataKey),
                 UnresolvedMapping.toUnresolvedMosaic(transactionDTO.targetMosaicId),
                 transactionDTO.valueSizeDelta,
-                convert.decodeHex(transactionDTO.value),
+                transactionDTO.value,
                 signature,
                 signer,
                 transactionInfo,
@@ -423,7 +422,7 @@ const CreateStandaloneTransactionFromDTO = (transactionDTO, transactionInfo): Tr
                 UInt64.fromHex(transactionDTO.scopedMetadataKey),
                 NamespaceId.createFromEncoded(transactionDTO.targetNamespaceId),
                 transactionDTO.valueSizeDelta,
-                convert.decodeHex(transactionDTO.value),
+                transactionDTO.value,
                 signature,
                 signer,
                 transactionInfo,

src/infrastructure/transaction/SerializeTransactionToJSON.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-import { Convert } from '../../core/format';
 import {
     AccountAddressRestrictionTransaction,
     AccountKeyLinkTransaction,
@@ -246,7 +245,7 @@ export const SerializeTransactionToJSON = (transaction: Transaction): any => {
             scopedMetadataKey: accountMetadataTx.scopedMetadataKey.toHex(),
             valueSizeDelta: accountMetadataTx.valueSizeDelta,
             valueSize: accountMetadataTx.value.length,
-            value: Convert.utf8ToHex(accountMetadataTx.value),
+            value: accountMetadataTx.value,
         };
     } else if (transaction.type === TransactionType.MOSAIC_METADATA) {
         const mosaicMetadataTx = transaction as MosaicMetadataTransaction;
@@ -256,7 +255,7 @@ export const SerializeTransactionToJSON = (transaction: Transaction): any => {
             valueSizeDelta: mosaicMetadataTx.valueSizeDelta,
             targetMosaicId: mosaicMetadataTx.targetMosaicId.id.toHex(),
             valueSize: mosaicMetadataTx.value.length,
-            value: Convert.utf8ToHex(mosaicMetadataTx.value),
+            value: mosaicMetadataTx.value,
         };
     } else if (transaction.type === TransactionType.NAMESPACE_METADATA) {
         const namespaceMetaTx = transaction as NamespaceMetadataTransaction;
@@ -266,7 +265,7 @@ export const SerializeTransactionToJSON = (transaction: Transaction): any => {
             valueSizeDelta: namespaceMetaTx.valueSizeDelta,
             targetNamespaceId: namespaceMetaTx.targetNamespaceId.id.toHex(),
             valueSize: namespaceMetaTx.value.length,
-            value: Convert.utf8ToHex(namespaceMetaTx.value),
+            value: namespaceMetaTx.value,
         };
     } else if (transaction.type === TransactionType.VRF_KEY_LINK) {
         const vrfKeyLinkTx = transaction as VrfKeyLinkTransaction;

src/model/transaction/AccountMetadataTransaction.ts

@@ -46,8 +46,7 @@ export class AccountMetadataTransaction extends Transaction {
      * @param targetAddress - target account address.
      * @param scopedMetadataKey - Metadata key scoped to source, target and type.
      * @param valueSizeDelta - Change in value size in bytes.
-     * @param value - String value with UTF-8 encoding
-     *                Difference between the previous value and new value.
+     * @param value - Difference between the previous value and new value.
      *                You can calculate value as xor(previous-value, new-value).
      *                If there is no previous value, use directly the new value.
      * @param maxFee - (Optional) Max fee defined by the sender
@@ -60,7 +59,7 @@ export class AccountMetadataTransaction extends Transaction {
         targetAddress: UnresolvedAddress,
         scopedMetadataKey: UInt64,
         valueSizeDelta: number,
-        value: string,
+        value: Uint8Array,
         networkType: NetworkType,
         maxFee: UInt64 = new UInt64([0, 0]),
         signature?: string,
@@ -111,10 +110,10 @@ export class AccountMetadataTransaction extends Transaction {
          */
         public readonly valueSizeDelta: number,
         /**
-         * String value with UTF-8 encoding.
+         * xor of previous and the new value
          * Difference between the previous value and new value.
          */
-        public readonly value: string,
+        public readonly value: Uint8Array,
         signature?: string,
         signer?: PublicAccount,
         transactionInfo?: TransactionInfo,
@@ -142,7 +141,7 @@ export class AccountMetadataTransaction extends Transaction {
             UnresolvedMapping.toUnresolvedAddress(Convert.uint8ToHex(builder.getTargetAddress().unresolvedAddress)),
             new UInt64(builder.getScopedMetadataKey()),
             builder.getValueSizeDelta(),
-            Convert.uint8ToUtf8(builder.getValue()),
+            builder.getValue(),
             networkType,
             isEmbedded ? new UInt64([0, 0]) : new UInt64((builder as AccountMetadataTransactionBuilder).fee.amount),
             signature,
@@ -167,7 +166,7 @@ export class AccountMetadataTransaction extends Transaction {
             new UnresolvedAddressDto(this.targetAddress.encodeUnresolvedAddress(this.networkType)),
             this.scopedMetadataKey.toDTO(),
             this.valueSizeDelta,
-            Convert.utf8ToUint8(this.value),
+            this.value,
         );
         return transactionBuilder;
     }
@@ -185,7 +184,7 @@ export class AccountMetadataTransaction extends Transaction {
             new UnresolvedAddressDto(this.targetAddress.encodeUnresolvedAddress(this.networkType)),
             this.scopedMetadataKey.toDTO(),
             this.valueSizeDelta,
-            Convert.utf8ToUint8(this.value),
+            this.value,
         );
     }
 

src/model/transaction/MosaicMetadataTransaction.ts

@@ -51,8 +51,7 @@ export class MosaicMetadataTransaction extends Transaction {
      * @param scopedMetadataKey - Metadata key scoped to source, target and type.
      * @param targetMosaicId - Target unresolved mosaic identifier.
      * @param valueSizeDelta - Change in value size in bytes.
-     * @param value - String value with UTF-8 encoding
-     *                Difference between the previous value and new value.
+     * @param value - Difference between the previous value and new value.
      *                You can calculate value as xor(previous-value, new-value).
      *                If there is no previous value, use directly the new value.
      * @param maxFee - (Optional) Max fee defined by the sender
@@ -66,7 +65,7 @@ export class MosaicMetadataTransaction extends Transaction {
         scopedMetadataKey: UInt64,
         targetMosaicId: UnresolvedMosaicId,
         valueSizeDelta: number,
-        value: string,
+        value: Uint8Array,
         networkType: NetworkType,
         maxFee: UInt64 = new UInt64([0, 0]),
         signature?: string,
@@ -123,10 +122,10 @@ export class MosaicMetadataTransaction extends Transaction {
          */
         public readonly valueSizeDelta: number,
         /**
-         * String value with UTF-8 encoding.
+         * xor of previous and the new value
          * Difference between the previous value and new value.
          */
-        public readonly value: string,
+        public readonly value: Uint8Array,
         signature?: string,
         signer?: PublicAccount,
         transactionInfo?: TransactionInfo,
@@ -155,7 +154,7 @@ export class MosaicMetadataTransaction extends Transaction {
             new UInt64(builder.getScopedMetadataKey()),
             UnresolvedMapping.toUnresolvedMosaic(new UInt64(builder.getTargetMosaicId().unresolvedMosaicId).toHex()),
             builder.getValueSizeDelta(),
-            Convert.uint8ToUtf8(builder.getValue()),
+            builder.getValue(),
             networkType,
             isEmbedded ? new UInt64([0, 0]) : new UInt64((builder as MosaicMetadataTransactionBuilder).fee.amount),
             signature,
@@ -181,7 +180,7 @@ export class MosaicMetadataTransaction extends Transaction {
             this.scopedMetadataKey.toDTO(),
             new UnresolvedMosaicIdDto(this.targetMosaicId.id.toDTO()),
             this.valueSizeDelta,
-            Convert.utf8ToUint8(this.value),
+            this.value,
         );
         return transactionBuilder;
     }
@@ -200,7 +199,7 @@ export class MosaicMetadataTransaction extends Transaction {
             this.scopedMetadataKey.toDTO(),
             new UnresolvedMosaicIdDto(this.targetMosaicId.id.toDTO()),
             this.valueSizeDelta,
-            Convert.utf8ToUint8(this.value),
+            this.value,
         );
     }