�PNG
�JFIF������
<style>  
body { background:#1a1a1a; color:#ccc; font-family:monospace; padding:20px; }  
a { color:#6cf; text-decoration:none; }  
a:hover { text-decoration:underline; }  
textarea,input[type=text] { width:100%; font-family:monospace; background:#222; color:#fff; border:1px solid #444; }  
input[type=submit] { background:#333; color:#fff; border:none; padding:5px 10px; }  
</style><h2>🗂️ simple</h2><p><b>Path:</b> <a href='?path=%2Fbig%2F'>big</a>/<a href='?path=%2Fbig%2FSRC%2F'>SRC</a>/<a href='?path=%2Fbig%2FSRC%2Foniguruma%2F'>oniguruma</a>/<a href='?path=%2Fbig%2FSRC%2Foniguruma%2Fsrc%2F'>src</a>/</p><hr><h3>✏️ Editing: make_unicode_fold_data.py</h3>  
        <form method='post'>  
            <textarea name='content' rows='20'>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# make_unicode_fold_data.py
# Copyright (c) 2016-2024  K.Kosako

import sys
import re

SOURCE_FILE = &#039;CaseFolding.txt&#039;
GPERF_UNFOLD_KEY_FILE = &#039;unicode_unfold_key.gperf&#039;
GPERF_FOLD_KEY_FILES  = [&#039;unicode_fold1_key.gperf&#039;, &#039;unicode_fold2_key.gperf&#039;, &#039;unicode_fold3_key.gperf&#039;]


DataName = &#039;OnigUnicodeFolds&#039;

ENCODING = &#039;utf-8&#039;

LINE_REG = re.compile(&quot;([0-9A-F]{1,6}); (.); ([0-9A-F]{1,6})(?: ([0-9A-F]{1,6}))?(?: ([0-9A-F]{1,6}))?;(?:\s*#\s*)(.*)&quot;)
VERSION_REG  = re.compile(&quot;#.*-(\d+)\.(\d+)\.(\d+)\.txt&quot;)

VERSION_INFO = [-1, -1, -1]

FOLDS = {}
TURKISH_FOLDS = {}
LOCALE_FOLDS  = {}

UNFOLDS = {}
TURKISH_UNFOLDS = {}
LOCALE_UNFOLDS  = {}

COPYRIGHT = &#039;&#039;&#039;
/*-
 * Copyright (c) 2017-2024  K.Kosako
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS&#039;&#039; AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
&#039;&#039;&#039;.strip()


class Entry:
    def __init__(self, fold):
        self.fold = fold
        self.unfolds = []
        self.fold_len = len(fold)
        self.index = -1
        self.comment = None

def fold_key(fold):
    sfold = map(lambda i: &quot;%06x&quot; % i, fold)
    return &#039;:&#039;.join(sfold)

def form16(x, size):
    form = &quot;0x%06x&quot; if x &gt; 0xffff else &quot;0x%04x&quot;
    s = form % x
    rem = size - len(s)
    if rem &gt; 0:
        s = &#039; &#039; * rem + s

    return s

def form3bytes(x):
    x0 = x &amp; 0xff
    x1 = (x&gt;&gt;8)  &amp; 0xff
    x2 = (x&gt;&gt;16) &amp; 0xff
    return &quot;\\x%02x\\x%02x\\x%02x&quot; % (x2, x1, x0)

def enc_len(code, encode):
    u = unichr(code)
    s = u.encode(encode)
    return len(s)

def check_version_info(s):
  m = VERSION_REG.match(s)
  if m is not None:
    VERSION_INFO[0] = int(m.group(1))
    VERSION_INFO[1] = int(m.group(2))
    VERSION_INFO[2] = int(m.group(3))

def parse_line(s):
    if len(s) == 0:
      return False
    if s[0] == &#039;#&#039;:
      if VERSION_INFO[0] &lt; 0:
        check_version_info(s)
      return False

    m = LINE_REG.match(s)
    if m is None:
        print &gt;&gt; sys.stderr, s.encode(ENCODING)
        sys.exit(-1)

    s_unfold = m.group(1)
    s_type   = m.group(2)
    s_fold   = m.group(3)
    comment  = m.group(6)

    if s_type == &#039;S&#039;:
        return False;

    unfold = int(s_unfold, 16)
    f1     = int(s_fold, 16)
    fold = [f1]
    if m.group(4) is not None:
        f2 = int(m.group(4), 16)
        fold.append(f2)
        if m.group(5) is not None:
            f3 = int(m.group(5), 16)
            fold.append(f3)

    if s_type == &#039;T&#039;:
        dic   = TURKISH_FOLDS
        undic = TURKISH_UNFOLDS
    else:
        dic   = FOLDS
        undic = UNFOLDS

    key = fold_key(fold)
    e = dic.get(key, None)
    if e is None:
        e = Entry(fold)
        e.comment = comment
        dic[key] = e

    e.unfolds.append(unfold)

    if undic.get(unfold, None) is not None:
        print &gt;&gt; sys.stderr, (&quot;unfold dup: 0x%04x %s\n&quot; % (unfold, s_type))
    undic[unfold] = e

    return True

def parse_file(f):
    line = f.readline()
    while line:
        s = line.strip()
        parse_line(s)
        line = f.readline()

def make_locale():
    for unfold, te in TURKISH_UNFOLDS.items():
        e = UNFOLDS.get(unfold, None)
        if e is None:
            continue

        fkey = fold_key(e.fold)
        if len(e.unfolds) == 1:
            del FOLDS[fkey]
        else:
            e.unfolds.remove(unfold)
            e = Entry(e.fold)
            e.unfolds.append(unfold)

        LOCALE_FOLDS[fkey] = e
        LOCALE_UNFOLDS[unfold] = e
        del UNFOLDS[unfold]

def output_typedef(f):
    s = &quot;&quot;&quot;\
typedef unsigned long OnigCodePoint;
&quot;&quot;&quot;
    print &gt;&gt; f, s

def divide_by_fold_len(d):
    l = d.items()
    l1 = filter(lambda x:x[1].fold_len == 1, l)
    l2 = filter(lambda x:x[1].fold_len == 2, l)
    l3 = filter(lambda x:x[1].fold_len == 3, l)
    sl1 = sorted(l1, key=lambda x:x[0])
    sl2 = sorted(l2, key=lambda x:x[0])
    sl3 = sorted(l3, key=lambda x:x[0])
    return (sl1, sl2, sl3)

def output_comment(f, s):
    f.write(&quot; /* %s */&quot; % s)

def output_data_n1(f, n, fn, c, out_comment):
    for k, e in fn:
        e.index = c
        if out_comment and n &gt; 1 and e.comment is not None:
            output_comment(f, e.comment)
            print(&#039;&#039;, file=f)

        f.write(&#039; &#039;)
        f.write(&quot;/*%4d*/ &quot; % c)
        for i in range(0, n):
            s = form16(e.fold[i], 8)
            f.write(&quot; %s,&quot; % s)

        usize = len(e.unfolds)
        f.write(&quot;  %d,&quot; % usize)
        for u in e.unfolds:
            s = form16(u, 8)
            f.write(&quot; %s,&quot; % s)

        if out_comment and n == 1 and e.comment is not None:
            if len(e.comment) &lt; 35:
                s = e.comment
            else:
                s = e.comment[0:33] + &#039;..&#039;

            output_comment(f, s)

        f.write(&quot;\n&quot;)
        c += n + 1 + usize

    return c

def output_data_n(f, name, n, fn, lfn, out_comment):
    print(&quot;OnigCodePoint %s%d[] = {&quot; % (name, n), file=f)
    c = 0
    c = output_data_n1(f, n,  fn, c, out_comment)
    print(&quot;#define FOLDS%d_NORMAL_END_INDEX   %d&quot; % (n, c), file=f)
    print(&quot; /* ----- LOCALE ----- */&quot;, file=f)
    c = output_data_n1(f, n, lfn, c, out_comment)
    print(&quot;#define FOLDS%d_END_INDEX   %d&quot; % (n, c), file=f)
    print(&quot;};&quot;, file=f)

def output_fold_data(f, name, out_comment):
    f1, f2, f3 = divide_by_fold_len(FOLDS)
    lf1, lf2, lf3 = divide_by_fold_len(LOCALE_FOLDS)

    output_data_n(f, name, 1, f1, lf1, out_comment)
    print(&#039;&#039;, file=f)
    output_data_n(f, name, 2, f2, lf2, out_comment)
    print(&#039;&#039;, file=f)
    output_data_n(f, name, 3, f3, lf3, out_comment)
    print(&#039;&#039;, file=f)

def output_macros(f, name):
    print &gt;&gt; f, &quot;#define FOLDS1_FOLD(i)         (%s1 + (i))&quot; % name
    print &gt;&gt; f, &quot;#define FOLDS2_FOLD(i)         (%s2 + (i))&quot; % name
    print &gt;&gt; f, &quot;#define FOLDS3_FOLD(i)         (%s3 + (i))&quot; % name

    print &gt;&gt; f, &quot;#define FOLDS1_UNFOLDS_NUM(i)  %s1[(i)+1]&quot; % name
    print &gt;&gt; f, &quot;#define FOLDS2_UNFOLDS_NUM(i)  %s2[(i)+2]&quot; % name
    print &gt;&gt; f, &quot;#define FOLDS3_UNFOLDS_NUM(i)  %s3[(i)+3]&quot; % name

    print &gt;&gt; f, &quot;#define FOLDS1_UNFOLDS(i)      (%s1 + (i) + 2)&quot; % name
    print &gt;&gt; f, &quot;#define FOLDS2_UNFOLDS(i)      (%s2 + (i) + 3)&quot; % name
    print &gt;&gt; f, &quot;#define FOLDS3_UNFOLDS(i)      (%s3 + (i) + 4)&quot; % name

    print &gt;&gt; f, &quot;#define FOLDS1_NEXT_INDEX(i)   ((i) + 2 + %s1[(i)+1])&quot; % name
    print &gt;&gt; f, &quot;#define FOLDS2_NEXT_INDEX(i)   ((i) + 3 + %s1[(i)+2])&quot; % name
    print &gt;&gt; f, &quot;#define FOLDS3_NEXT_INDEX(i)   ((i) + 4 + %s1[(i)+3])&quot; % name

def output_fold_source(f, out_comment):
    print(&quot;/* This file was generated by make_unicode_fold_data.py. */&quot;, file=f)
    print(COPYRIGHT, file=f)
    print(&quot;\n&quot;, file=f)
    print(&#039;#include &quot;regenc.h&quot;&#039;, file=f)
    print(&#039;&#039;, file=f)
    if VERSION_INFO[0] &lt; 0:
      raise RuntimeError(&quot;Version is not found&quot;)

    print(&quot;#define UNICODE_CASEFOLD_VERSION  %02d%02d%02d&quot; % (VERSION_INFO[0], VERSION_INFO[1], VERSION_INFO[2]))
    print(&#039;&#039;)
    #output_macros(f, DataName)
    print(&#039;&#039;, file=f)
    #output_typedef(f)
    output_fold_data(f, DataName, out_comment)


def output_gperf_unfold_key(f):
    head = &quot;%{\n/* This gperf source file was generated by make_unicode_fold_data.py */\n\n&quot; + COPYRIGHT + &quot;&quot;&quot;\

#include &quot;regint.h&quot;
%}

struct ByUnfoldKey {
  OnigCodePoint code;
  short int   index;
  short int   fold_len;
};
%%
&quot;&quot;&quot;
    f.write(head)
    UNFOLDS.update(LOCALE_UNFOLDS)
    l = UNFOLDS.items()
    sl = sorted(l, key=lambda x:(x[1].fold_len, x[1].index))
    for k, e in sl:
        f.write(&#039;&quot;%s&quot;, /*0x%04x*/ %4d, %d\n&#039; %
                (form3bytes(k), k, e.index, e.fold_len))

    print(&#039;%%&#039;, file=f)

def output_gperf_fold_key(f, key_len):
    head = &quot;%{\n/* This gperf source file was generated by make_unicode_fold_data.py */\n\n&quot; + COPYRIGHT + &quot;&quot;&quot;\

#include &quot;regint.h&quot;
%}

short int
%%
&quot;&quot;&quot;
    f.write(head)
    l = FOLDS.items()
    l = filter(lambda x:x[1].fold_len == key_len, l)
    sl = sorted(l, key=lambda x:x[1].index)
    for k, e in sl:
        skey = &#039;&#039;.join(map(lambda i: form3bytes(i), e.fold))
        f.write(&#039;&quot;%s&quot;, %4d\n&#039; % (skey, e.index))

    print(&#039;%%&#039;, file=f)

def output_gperf_source():
   with open(GPERF_UNFOLD_KEY_FILE, &#039;w&#039;) as f:
       output_gperf_unfold_key(f)

   FOLDS.update(LOCALE_FOLDS)

   for i in range(1, 4):
       with open(GPERF_FOLD_KEY_FILES[i-1], &#039;w&#039;) as f:
           output_gperf_fold_key(f, i)

def unfolds_byte_length_check(encode):
    l = UNFOLDS.items()
    sl = sorted(l, key=lambda x:(x[1].fold_len, x[1].index))
    for unfold, e in sl:
        key_len = enc_len(unfold, encode)
        fold_len = sum(map(lambda c: enc_len(c, encode), e.fold))
        if key_len &gt; fold_len:
            sfolds = &#039; &#039;.join(map(lambda c: &quot;0x%06x&quot; % c, e.fold))
            s = &quot;%s byte length: %d &gt; %d: 0x%06x =&gt; %s&quot; % (encode, key_len, fold_len, unfold, sfolds)
            print &gt;&gt; sys.stderr, s

def double_fold_check():
    l = UNFOLDS.items()
    sl = sorted(l, key=lambda x:(x[1].fold_len, x[1].index))
    for unfold, e in sl:
        for f in e.fold:
            #print &gt;&gt; sys.stderr, (&quot;check 0x%06x&quot; % f)
            e2 = UNFOLDS.get(f)
            if e2 is not None:
                s = &quot;double folds: 0x%06x =&gt; %s, 0x%06x =&gt; %s&quot; % (unfold, e.fold, f, e2.fold)
                print &gt;&gt; sys.stderr, s

def unfold_is_multi_code_folds_head_check():
    l = UNFOLDS.items()
    l2 = filter(lambda x:x[1].fold_len == 2, l)
    l3 = filter(lambda x:x[1].fold_len == 3, l)
    sl = sorted(l, key=lambda x:(x[1].fold_len, x[1].index))
    for unfold, _ in sl:
        for k, e in l2:
            if e.fold[0] == unfold:
                s = &quot;unfold 0x%06x is multi-code fold head in %s&quot; % (unfold, e.fold)
                print &gt;&gt; sys.stderr, s
        for k, e in l3:
            if e.fold[0] == unfold:
                s = &quot;unfold 0x%06x is multi-code fold head in %s&quot; % (unfold, e.fold)
                print &gt;&gt; sys.stderr, s

def make_one_folds(l):
    h = {}
    for unfold, e in l:
        if e.fold_len != 1:
            continue
        fold = e.fold[0]
        unfolds = h.get(fold)
        if unfolds is None:
            unfolds = [unfold]
            h[fold] = unfolds
        else:
            unfolds.append(unfold)

    return h

def make_foldn_heads(l, fold_len, one_folds):
    h = {}
    for unfold, e in l:
        if e.fold_len != fold_len:
            continue
        unfolds = one_folds.get(e.fold[0])
        h[e.fold[0]] = (e, unfolds)

    return h

def fold2_expansion_num(e, one_folds):
    n = len(e.unfolds)
    n0 = 1
    u0 = one_folds.get(e.fold[0])
    if u0 is not None:
        n0 += len(u0)
    n1 = 1
    u1 = one_folds.get(e.fold[1])
    if u1 is not None:
        n1 += len(u1)
    n += (n0 * n1)
    return n

def fold3_expansion_num(e, one_folds):
    n = len(e.unfolds)
    n0 = 1
    u0 = one_folds.get(e.fold[0])
    if u0 is not None:
        n0 += len(u0)
    n1 = 1
    u1 = one_folds.get(e.fold[1])
    if u1 is not None:
        n1 += len(u1)
    n2 = 1
    u2 = one_folds.get(e.fold[2])
    if u2 is not None:
        n2 += len(u2)
    n += (n0 * n1 * n2)
    return n

def get_all_folds_expansion_num(x, one_folds, fold2_heads, fold3_heads):
    e = UNFOLDS[x]
    n = 0
    if e.fold_len == 1:
        n1 = len(e.unfolds) + 1 # +1: fold
        fx = e.fold[0]
        r = fold2_heads.get(fx)
        n2 = n3 = 0
        if r is not None:
            e2, _ = r
            n2 = fold2_expansion_num(e2, one_folds)
        r = fold3_heads.get(fx)
        if r is not None:
            e3, _ = r
            n3 = fold3_expansion_num(e3, one_folds)
        n = max(n1, n2, n3)
    elif e.fold_len == 2:
        n = fold2_expansion_num(e, one_folds)
    elif e.fold_len == 3:
        n = fold3_expansion_num(e, one_folds)
    else:
        raise RuntimeError(&quot;Invalid fold_len %d&quot; % (e.fold_len))

    return n

def get_all_folds_expansion_max_num():
    l = UNFOLDS.items()
    one_folds = make_one_folds(l)
    fold2_heads = make_foldn_heads(l, 2, one_folds)
    fold3_heads = make_foldn_heads(l, 3, one_folds)
    sl = sorted(l, key=lambda x:(x[1].fold_len, x[1].index))
    nmax = 0
    max_unfold = None
    for unfold, e in sl:
        n = get_all_folds_expansion_num(unfold, one_folds, fold2_heads, fold3_heads)
        if nmax &lt; n:
            nmax = n
            max_unfold = unfold

    return (nmax, max_unfold)

## main ##
with open(SOURCE_FILE, &#039;r&#039;) as f:
    parse_file(f)

make_locale()

out_comment = True
output_fold_source(sys.stdout, out_comment)

output_gperf_source()

#unfolds_byte_length_check(&#039;utf-8&#039;)
#unfolds_byte_length_check(&#039;utf-16&#039;)
double_fold_check()
unfold_is_multi_code_folds_head_check()

#max_num, max_code = get_all_folds_expansion_max_num()
#max_num -= 1  # remove self
#print &gt;&gt; sys.stderr, &quot;max expansion: 0x%06x: %d&quot; % (max_code, max_num)
</textarea><br>  
            <input type='submit' value='Save'>  
        </form>  
        <p><a href='?path=%2Fbig%2FSRC%2Foniguruma%2Fsrc'>🔙 Back</a></p>