use tokenizer from mlnet

Program.cs

@@ -1,35 +1,28 @@
-using System;
+using Microsoft.ML;
+using Microsoft.ML.Tokenizers;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using TorchSharp;
 
-torchvision.io.DefaultImager = new torchvision.io.SkiaImager();
-var device = TorchSharp.torch.device("cuda:0");
-var clipEncoder = new ClipEncoder("clip_encoder.ckpt", device);
+var batch = 1;
+var bpe = new Bpe("vocab.json", "merges.txt", endOfWordSuffix: "</w>");
+var tokenier = new Tokenizer(bpe);
 var start_token = 49406;
 var end_token = 49407;
-var dictionary = new Dictionary<string, long>(){
-    {"cat", 2368},
-    {"a", 320},
-    {"cute", 2242},
-    {"blue", 1746},
-    {"wild", 3220},
-    {"green", 1901},
-};
-
-var batch = 1;
-
 var prompt = "a wild cute green cat";
-var tokens = prompt.Split(' ').Select(x => dictionary[x]).ToList();
-tokens = tokens.Prepend(start_token).ToList();
-tokens = tokens.Append(end_token).ToList();
-tokens = tokens.Concat(Enumerable.Repeat<long>(0, 77 - tokens.Count)).ToList();
+var res = tokenier.Encode(prompt);
+var tokens = new[] { start_token }.Concat(res.Ids.Concat(Enumerable.Repeat(0, 75 - res.Ids.Count))).Concat(new[] { end_token }).ToList();
 var uncontional_tokens = new[]{start_token, end_token}.Concat(Enumerable.Repeat(0, 75)).ToList();
 var tokenTensor = torch.tensor(tokens.ToArray(), dtype: torch.ScalarType.Int64, device: device);
 tokenTensor = tokenTensor.repeat(batch, 1);
 var unconditional_tokenTensor = torch.tensor(uncontional_tokens.ToArray(), dtype: torch.ScalarType.Int64, device: device);
 unconditional_tokenTensor = unconditional_tokenTensor.repeat(batch, 1);
+
+torchvision.io.DefaultImager = new torchvision.io.SkiaImager();
+var device = TorchSharp.torch.device("cuda:0");
+var clipEncoder = new ClipEncoder("clip_encoder.ckpt", device);
 var img = torch.randn(batch, 4, 64, 64, dtype: torch.ScalarType.Float32, device: device);
 var t = torch.full(new[]{batch, 1L}, value: batch, dtype: torch.ScalarType.Int32, device: device);
 var condition = clipEncoder.Forward(tokenTensor);

clip.csproj

@@ -9,12 +9,19 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.ML" Version="2.0.1" />
+    <PackageReference Include="Microsoft.ML.Tokenizers" Version="0.20.1" />
     <PackageReference Include="TorchVision" Version="$(TorchVersion)" />
     <PackageReference Include="TorchSharp-cuda-linux" Version="$(TorchVersion)" />
-     <None Update="*.ckpt">
+    <None Update="*.ckpt">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="merges.txt">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+    <None Update="vocab.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
-    <PackageReference Include="Microsoft.ML" Version="2.0.1" />
   </ItemGroup>
 
 </Project>