Update Jenkinsfile

添加网易前端过滤敏感词库

.dockerignore

@@ -0,0 +1,27 @@
+.git
+.github
+.idea
+.vscode
+Dockerfile
+.dockerignore
+
+# Build outputs
+bin/
+build/
+dist/
+out/
+
+# Archives & temp
+*.zip
+*.tar
+*.tar.gz
+*.rar
+*.7z
+*.log
+*.tmp
+*.swp
+~*
+
+# Optional: exclude non-runtime assets
+Organized/
+ThirdPartyCompatibleFormats/

.github/workflows/main.yml

@@ -4,12 +4,6 @@ on:
   push:
     tags:
       - '*'
-  workflow_dispatch:
-    inputs:
-      tag_name:
-        description: '请输入 tag 名（如 1.0）'
-        required: true
-        default: '1.0'
 
 jobs:
   release:

.github/workflows/server.yml

@@ -0,0 +1,132 @@
+name: Server CI/CD
+
+on:
+  push:
+    branches: [ "dev", "master" ]
+    tags:
+      - '*'
+  pull_request:
+    branches: [ "dev", "master" ]
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build:
+    name: Build server binaries
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [ linux, windows ]
+        goarch: [ amd64 ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.22.x'
+          cache: true
+
+      - name: Download modules
+        run: go mod download
+
+      - name: Vet & Test
+        run: |
+          go vet ./...
+          go test ./... -v
+
+      - name: Build cmd/server
+        shell: bash
+        run: |
+          mkdir -p build
+          EXT=""
+          if [ "${{ matrix.goos }}" = "windows" ]; then EXT=".exe"; fi
+          OUT="server-${{ matrix.goos }}-${{ matrix.goarch }}${EXT}"
+          echo "Building $OUT"
+          GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }} \
+            go build -trimpath -ldflags="-s -w" -o "build/${OUT}" ./cmd/server
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: server-${{ matrix.goos }}-${{ matrix.goarch }}
+          path: build/server-${{ matrix.goos }}-${{ matrix.goarch }}*
+
+  release:
+    name: Release binaries
+    needs: build
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/')
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+
+      - name: List artifacts
+        run: ls -R dist
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: dist/**/*
+          draft: false
+          prerelease: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  docker:
+    name: Build and push Docker image
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Determine image name and tags
+        id: imagetags
+        shell: bash
+        run: |
+          IMAGE="ghcr.io/${{ github.repository_owner }}/sensitive-lexicon-server"
+          echo "IMAGE=$IMAGE" >> $GITHUB_ENV
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            echo "PUSH=false" >> $GITHUB_ENV
+            echo "TAGS=${IMAGE}:pr-${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+          elif [ "${{ github.ref_type }}" = "tag" ]; then
+            echo "PUSH=true" >> $GITHUB_ENV
+            echo "TAGS=${IMAGE}:${{ github.ref_name }},${IMAGE}:latest" >> $GITHUB_ENV
+          else
+            BRANCH="${{ github.ref_name }}"
+            echo "PUSH=true" >> $GITHUB_ENV
+            if [ "$BRANCH" = "dev" ] || [ "$BRANCH" = "master" ]; then
+              echo "TAGS=${IMAGE}:latest,${IMAGE}:sha-${{ github.sha }}" >> $GITHUB_ENV
+            else
+              echo "TAGS=${IMAGE}:branch-${BRANCH},${IMAGE}:sha-${{ github.sha }}" >> $GITHUB_ENV
+            fi
+          fi
+          echo "Using tags: $TAGS"
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: ${{ env.PUSH }}
+          tags: ${{ env.TAGS }}

.gitignore

@@ -0,0 +1,45 @@
+# Go build artifacts
+bin/
+build/
+dist/
+out/
+
+# Executables and shared libs
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test and coverage outputs
+*.test
+coverage.out
+*.coverprofile
+
+# Logs
+*.log
+logs/
+
+# Env files
+.env
+.env.*
+
+# IDE settings
+.vscode/
+.idea/
+*.iml
+
+# OS files
+.DS_Store
+Thumbs.db
+
+# Archives and temporary files
+*.zip
+*.tar
+*.tar.gz
+*.rar
+*.7z
+*.tmp
+*.temp
+*.swp
+~*

Dockerfile

@@ -0,0 +1,37 @@
+# syntax=docker/dockerfile:1.4
+FROM golang:1.22-alpine AS builder
+WORKDIR /src
+
+# Pre-fetch deps
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy source
+COPY . .
+
+# Normalize and tidy modules inside build context
+RUN go mod tidy
+
+# Build static binary for target platform
+ARG TARGETOS
+ARG TARGETARCH
+ENV CGO_ENABLED=0
+RUN GOOS=$TARGETOS GOARCH=$TARGETARCH \
+    go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
+
+FROM gcr.io/distroless/static:nonroot
+WORKDIR /app
+
+# App binary
+COPY --from=builder /out/server /app/server
+
+# Default lexicon files
+COPY Vocabulary /app/Vocabulary
+
+# Default envs
+ENV PORT=8080
+ENV LEXICON_DIR=Vocabulary
+
+EXPOSE 8080
+USER nonroot
+ENTRYPOINT ["/app/server"]

Jenkinsfile

@@ -0,0 +1,98 @@
+pipeline {
+  agent {label 'dockeragent'}
+  // 构建逻辑已迁移到 Dockerfile，Jenkins 不再进行本地 go build
+
+  environment {
+    GO111MODULE = 'on'        // 开启 Modules 模式
+    CGO_ENABLED = '0'
+    APP_NAME = 'sensitive-lexicon'
+    REGISTRY = 'crpi-vqe38j3xeblrq0n4.cn-hangzhou.personal.cr.aliyuncs.com/go-mctown'
+  }
+
+  stages {
+    stage('Checkout') {
+      steps {
+        checkout scm
+      }
+    }
+
+    // 使用 Dockerfile 完成编译与打包，仅保留镜像构建与推送
+    stage('Docker Build & Push') {
+      steps {
+        withCredentials([usernamePassword(
+          credentialsId: 'aliyun-docker-login',
+          usernameVariable: 'DOCKER_USERNAME',
+          passwordVariable: 'DOCKER_PASSWORD'
+        )]) {
+          sh """
+            echo "\$DOCKER_PASSWORD" | docker login --username \$DOCKER_USERNAME --password-stdin ${env.REGISTRY.split('/')[0]}
+          """
+        }
+
+        script {
+          def imageTag = "${env.REGISTRY}/${env.APP_NAME}:${env.BUILD_NUMBER}"
+          def latestTag = "${env.REGISTRY}/${env.APP_NAME}:latest"
+
+          sh """
+            ls -l
+            docker build -t ${imageTag} --network=host .
+            docker tag ${imageTag} ${latestTag}
+            docker push ${imageTag}
+            docker push ${latestTag}
+          """
+        }
+      }
+    }
+
+stage('Deploy All Compose Projects') {
+      parallel {
+        stage('Deploy compose1') {
+  agent {label 'dockeragent'}
+          steps {
+
+              checkout scm
+                              sh """
+                pwd
+                ls -l
+                """
+            dir('deploy/compose') {
+              script {
+                withCredentials([usernamePassword(
+                  credentialsId: 'aliyun-docker-login',
+                  usernameVariable: 'DOCKER_USERNAME',
+                  passwordVariable: 'DOCKER_PASSWORD'
+                )]) {
+                  sh """
+                    echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin ${env.REGISTRY.split('/')[0]}
+                  """
+                }
+                sh """
+                pwd
+                ls -l
+                docker compose -f docker-compose.yml down || true
+                docker compose -f docker-compose.yml pull
+                docker compose -f docker-compose.yml up -d --remove-orphans
+                """
+              }
+            }
+          }
+        }
+
+      }
+    }
+
+
+  }
+
+  post {
+    always {
+      cleanWs()
+    }
+    success {
+      echo "✅ 构建成功！"
+    }
+    failure {
+      echo "🔥 构建失败，请检查日志。"
+    }
+  }
+}

Organized/政治类型.txt

@@ -0,0 +1 @@
+

README.md

@@ -33,6 +33,7 @@ Sensitive‑lexicon 提供了一份广泛覆盖政治、色情、暴力等敏感
 ```
 Sensitive-lexicon/
 ├── ThirdPartyCompatibleFormats/        # 用于第三方格式
+├── Organized/                          # 已经进行整理的词库
 ├── Vocabulary/                         # 词汇库
 ├── LICENSE                             # 许可证
 └── README.md                           # 项目说明
@@ -43,7 +44,7 @@ Sensitive-lexicon/
 ### 集成到项目
 
 1. 克隆或下载本仓库。
-2. 在您的代码中读取 `sensitive-lexicon.txt`（或您需要的分支文件）。
+2. 在您的代码中读取 `词库中的 .txt 文件`（或您需要的分支文件）。
 3. 根据业务场景，选择合适的匹配算法（如 DFA、Trie、正则表达式等）进行过滤。
 
 ```bash
@@ -82,3 +83,21 @@ git clone https://github.com/Konsheng/Sensitive-lexicon.git
     <img alt="Star History Chart" src="https://api.star-history.com/svg?repos=konsheng/Sensitive-lexicon&type=Date" />
   </picture>
 </a>
+
+## 运行敏感词检测服务（Fiber + fuzzy-patricia）
+
+```bash
+# Windows PowerShell 示例
+$env:PORT="8080"; $env:LEXICON_DIR="Vocabulary"; $env:FUZZY_MAX_DISTANCE="1"
+
+# 构建并运行
+go mod tidy
+go build -o bin\server.exe ./cmd/server
+./bin/server.exe
+```
+
+- POST `/detect`
+  - 请求体: `{ "text": "待检测文本", "enable_fuzzy": true }`
+  - 响应: `{ "hits": [{"word":"...","type":"substring|fuzzy","distance":0}] }`
+- POST `/reload` 重新加载 `Vocabulary` 目录
+- GET `/health` 存活探针

Vocabulary/其他词库.txt

@@ -1,3 +1,4 @@
+力工
 穴海
 协警
 纳米比亚

Vocabulary/政治类型.txt

@@ -1,326 +1,326 @@
-习近平,
+习近平
-平近习,
+平近习
-xjp,
+xjp
-习太子,
+习太子
-习明泽,
+习明泽
-老习,
+老习
-温家宝,
+温家宝
-温加宝,
+温加宝
-温x,
+温x
-温jia宝,
+温jia宝
-温宝宝,
+温宝宝
-温加饱,
+温加饱
-温加保,
+温加保
-张培莉,
+张培莉
-温云松,
+温云松
-温如春,
+温如春
-温jb,
+温jb
-胡温,
+胡温
-胡x,
+胡x
-胡jt,
+胡jt
-胡boss,
+胡boss
-胡总,
+胡总
-胡王八,
+胡王八
-hujintao,
+hujintao
-胡jintao,
+胡jintao
-胡j涛,
+胡j涛
-胡惊涛,
+胡惊涛
-胡景涛,
+胡景涛
-胡紧掏,
+胡紧掏
-湖紧掏,
+湖紧掏
-胡紧套,
+胡紧套
-锦涛,
+锦涛
-hjt,
+hjt
-胡派,
+胡派
-胡主席,
+胡主席
-刘永清,
+刘永清
-胡海峰,
+胡海峰
-胡海清,
+胡海清
-江泽民,
+江泽民
-民泽江,
+民泽江
-江胡,
+江胡
-江哥,
+江哥
-江主席,
+江主席
-江书记,
+江书记
-江浙闽,
+江浙闽
-江沢民,
+江沢民
-江浙民,
+江浙民
-择民,
+择民
-则民,
+则民
-茳泽民,
+茳泽民
-zemin,
+zemin
-ze民,
+ze民
-老江,
+老江
-老j,
+老j
-江core,
+江core
-江x,
+江x
-江派,
+江派
-江zm,
+江zm
-jzm,
+jzm
-江戏子,
+江戏子
-江蛤蟆,
+江蛤蟆
-江某某,
+江某某
-江贼,
+江贼
-江猪,
+江猪
-江氏集团,
+江氏集团
-江绵恒,
+江绵恒
-江绵康,
+江绵康
-王冶坪,
+王冶坪
-江泽慧,
+江泽慧
-邓小平,
+邓小平
-平小邓,
+平小邓
-xiao平,
+xiao平
-邓xp,
+邓xp
-邓晓平,
+邓晓平
-邓朴方,
+邓朴方
-邓榕,
+邓榕
-邓质方,
+邓质方
-毛泽东,
+毛泽东
-猫泽东,
+猫泽东
-猫则东,
+猫则东
-猫贼洞,
+猫贼洞
-毛zd,
+毛zd
-毛zx,
+毛zx
-z东,
+z东
-ze东,
+ze东
-泽d,
+泽d
-zedong,
+zedong
-毛太祖,
+毛太祖
-毛相,
+毛相
-主席画像,
+主席画像
-改革历程,
+改革历程
-朱镕基,
+朱镕基
-朱容基,
+朱容基
-朱镕鸡,
+朱镕鸡
-朱容鸡,
+朱容鸡
-朱云来,
+朱云来
-李鹏,
+李鹏
-李peng,
+李peng
-里鹏,
+里鹏
-李月月鸟,
+李月月鸟
-李小鹏,
+李小鹏
-李小琳,
+李小琳
-华主席,
+华主席
-华国,
+华国
-国锋,
+国锋
-国峰,
+国峰
-锋同志,
+锋同志
-白春礼,
+白春礼
-薄熙来,
+薄熙来
-薄一波,
+薄一波
-蔡赴朝,
+蔡赴朝
-蔡武,
+蔡武
-曹刚川,
+曹刚川
-常万全,
+常万全
-陈炳德,
+陈炳德
-陈德铭,
+陈德铭
-陈建国,
+陈建国
-陈良宇,
+陈良宇
-陈绍基,
+陈绍基
-陈同海,
+陈同海
-陈至立,
+陈至立
-戴秉国,
+戴秉国
-丁一平,
+丁一平
-董建华,
+董建华
-杜德印,
+杜德印
-杜世成,
+杜世成
-傅锐,
+傅锐
-郭伯雄,
+郭伯雄
-郭金龙,
+郭金龙
-贺国强,
+贺国强
-胡春华,
+胡春华
-耀邦,
+耀邦
-华建敏,
+华建敏
-黄华华,
+黄华华
-黄丽满,
+黄丽满
-黄兴国,
+黄兴国
-回良玉,
+回良玉
-贾庆林,
+贾庆林
-贾廷安,
+贾廷安
-靖志远,
+靖志远
-李长春,
+李长春
-李春城,
+李春城
-李建国,
+李建国
-李克强,
+李克强
-李岚清,
+李岚清
-李沛瑶,
+李沛瑶
-李荣融,
+李荣融
-李瑞环,
+李瑞环
-李铁映,
+李铁映
-李先念,
+李先念
-李学举,
+李学举
-李源潮,
+李源潮
-栗智,
+栗智
-梁光烈,
+梁光烈
-廖锡龙,
+廖锡龙
-林树森,
+林树森
-林炎志,
+林炎志
-林左鸣,
+林左鸣
-令计划,
+令计划
-柳斌杰,
+柳斌杰
-刘奇葆,
+刘奇葆
-刘少奇,
+刘少奇
-刘延东,
+刘延东
-刘云山,
+刘云山
-刘志军,
+刘志军
-龙新民,
+龙新民
-路甬祥,
+路甬祥
-罗箭,
+罗箭
-吕祖善,
+吕祖善
-马飚,
+马飚
-马恺,
+马恺
-孟建柱,
+孟建柱
-欧广源,
+欧广源
-强卫,
+强卫
-沈跃跃,
+沈跃跃
-宋平顺,
+宋平顺
-粟戎生,
+粟戎生
-苏树林,
+苏树林
-孙家正,
+孙家正
-铁凝,
+铁凝
-屠光绍,
+屠光绍
-王东明,
+王东明
-汪东兴,
+汪东兴
-王鸿举,
+王鸿举
-王沪宁,
+王沪宁
-王乐泉,
+王乐泉
-王洛林,
+王洛林
-王岐山,
+王岐山
-王胜俊,
+王胜俊
-王太华,
+王太华
-王学军,
+王学军
-王兆国,
+王兆国
-王振华,
+王振华
-吴邦国,
+吴邦国
-吴定富,
+吴定富
-吴官正,
+吴官正
-无官正,
+无官正
-吴胜利,
+吴胜利
-吴仪,
+吴仪
-奚国华,
+奚国华
-习仲勋,
+习仲勋
-徐才厚,
+徐才厚
-许其亮,
+许其亮
-徐绍史,
+徐绍史
-杨洁篪,
+杨洁篪
-叶剑英,
+叶剑英
-由喜贵,
+由喜贵
-于幼军,
+于幼军
-俞正声,
+俞正声
-袁纯清,
+袁纯清
-曾培炎,
+曾培炎
-曾庆红,
+曾庆红
-曾宪梓,
+曾宪梓
-曾荫权,
+曾荫权
-张德江,
+张德江
-张定发,
+张定发
-张高丽,
+张高丽
-张立昌,
+张立昌
-张荣坤,
+张荣坤
-张志国,
+张志国
-赵洪祝,
+赵洪祝
-紫阳,
+紫阳
-周生贤,
+周生贤
-周永康,
+周永康
-朱海仑,
+朱海仑
-中南海,
+中南海
-大陆当局,
+大陆当局
-中国当局,
+中国当局
-北京当局,
+北京当局
-共产党,
+共产党
-党产共,
+党产共
-共贪党,
+共贪党
-阿共,
+阿共
-产党共,
+产党共
-公产党,
+公产党
-工产党,
+工产党
-共c党,
+共c党
-共x党,
+共x党
-共铲,
+共铲
-供产,
+供产
-共惨,
+共惨
-供铲党,
+供铲党
-供铲谠,
+供铲谠
-供铲裆,
+供铲裆
-共残党,
+共残党
-共残主义,
+共残主义
-共产主义的幽灵,
+共产主义的幽灵
-拱铲,
+拱铲
-老共,
+老共
-中共,
+中共
-中珙,
+中珙
-中gong,
+中gong
-gc党,
+gc党
-贡挡,
+贡挡
-gong党,
+gong党
-g产,
+g产
-狗产蛋,
+狗产蛋
-共残裆,
+共残裆
-恶党,
+恶党
-邪党,
+邪党
-共产专制,
+共产专制
-共产王朝,
+共产王朝
-裆中央,
+裆中央
-土共,
+土共
-土g,
+土g
-共狗,
+共狗
-g匪,
+g匪
-共匪,
+共匪
-仇共,
+仇共
-政府,
+政府
-症腐,
+症腐
-政腐,
+政腐
-政付,
+政付
-正府,
+正府
-政俯,
+政俯
-政f,
+政f
-zhengfu,
+zhengfu
-政zhi,
+政zhi
-挡中央,
+挡中央
-档中央,
+档中央
-中央领导,
+中央领导
-中国zf,
+中国zf
-中央zf,
+中央zf
-国wu院,
+国wu院
-中华帝国,
+中华帝国
-gong和,
+gong和
-大陆官方,
+大陆官方
-北京政权,
+北京政权
-江泽民,
+江泽民
-胡锦涛,
+胡锦涛
-温家宝,
+温家宝
-习近平,
+习近平
-习仲勋,
+习仲勋
-贺国强,
+贺国强
-贺子珍,
+贺子珍
-周永康,
+周永康
-李长春,
+李长春
-李德生,
+李德生
-王岐山,
+王岐山
-姚依林,
+姚依林
-回良玉,
+回良玉
-李源潮,
+李源潮
-李干成,
+李干成
-戴秉国,
+戴秉国
-黄镇,
+黄镇
-刘延东,
+刘延东
-刘瑞龙,
+刘瑞龙
-俞正声,
+俞正声
-黄敬,
+黄敬
-薄熙,
+薄熙
-薄一波,
+薄一波
-周小川,
+周小川
-周建南,
+周建南
-温云松,
+温云松
-徐明,
+徐明
-江泽慧,
+江泽慧
-江绵恒,
+江绵恒
-江绵康,
+江绵康
-李小鹏,
+李小鹏
-李鹏,
+李鹏
-李小琳,
+李小琳
-朱云来,
+朱云来
-朱容基,
+朱容基
-法轮功,
+法轮功
-李洪志,
+李洪志
 新疆骚乱

Vocabulary/色情类型.txt

@@ -1,304 +1,304 @@
-爱液,
+爱液
-按摩棒,
+按摩棒
-拔出来,
+拔出来
-爆草,
+爆草
-包二奶,
+包二奶
-暴干,
+暴干
-暴奸,
+暴奸
-暴乳,
+暴乳
-爆乳,
+爆乳
-暴淫,
+暴淫
-被操,
+被操
-被插,
+被插
-被干,
+被干
-逼奸,
+逼奸
-仓井空,
+仓井空
-插暴,
+插暴
-操逼,
+操逼
-操黑,
+操黑
-操烂,
+操烂
-肏你,
+肏你
-肏死,
+肏死
-操死,
+操死
-操我,
+操我
-厕奴,
+厕奴
-插比,
+插比
-插b,
+插b
-插逼,
+插逼
-插进,
+插进
-插你,
+插你
-插我,
+插我
-插阴,
+插阴
-潮吹,
+潮吹
-潮喷,
+潮喷
-成人电影,
+成人电影
-成人论坛,
+成人论坛
-成人色情,
+成人色情
-成人网站,
+成人网站
-成人文学,
+成人文学
-成人小说,
+成人小说
-艳情小说,
+艳情小说
-成人游戏,
+成人游戏
-吃精,
+吃精
-抽插,
+抽插
-春药,
+春药
-大波,
+大波
-大力抽送,
+大力抽送
-大乳,
+大乳
-荡妇,
+荡妇
-荡女,
+荡女
-盗撮,
+盗撮
-发浪,
+发浪
-放尿,
+放尿
-肥逼,
+肥逼
-粉穴,
+粉穴
-风月大陆,
+风月大陆
-干死你,
+干死你
-干穴,
+干穴
-肛交,
+肛交
-肛门,
+肛门
-龟头,
+龟头
-裹本,
+裹本
-国产av,
+国产av
-好嫩,
+好嫩
-豪乳,
+豪乳
-黑逼,
+黑逼
-后庭,
+后庭
-后穴,
+后穴
-虎骑,
+虎骑
-换妻俱乐部,
+换妻俱乐部
-黄片,
+黄片
-几吧,
+几吧
-鸡吧,
+鸡吧
-鸡巴,
+鸡巴
-鸡奸,
+鸡奸
-妓女,
+妓女
-奸情,
+奸情
-叫床,
+叫床
-脚交,
+脚交
-精液,
+精液
-就去日,
+就去日
-巨屌,
+巨屌
-菊花洞,
+菊花洞
-菊门,
+菊门
-巨奶,
+巨奶
-巨乳,
+巨乳
-菊穴,
+菊穴
-开苞,
+开苞
-口爆,
+口爆
-口活,
+口活
-口交,
+口交
-口射,
+口射
-口淫,
+口淫
-裤袜,
+裤袜
-狂操,
+狂操
-狂插,
+狂插
-浪逼,
+浪逼
-浪妇,
+浪妇
-浪叫,
+浪叫
-浪女,
+浪女
-狼友,
+狼友
-聊性,
+聊性
-凌辱,
+凌辱
-漏乳,
+漏乳
-露b,
+露b
-乱交,
+乱交
-乱伦,
+乱伦
-轮暴,
+轮暴
-轮操,
+轮操
-轮奸,
+轮奸
-裸陪,
+裸陪
-买春,
+买春
-美逼,
+美逼
-美少妇,
+美少妇
-美乳,
+美乳
-美腿,
+美腿
-美穴,
+美穴
-美幼,
+美幼
-秘唇,
+秘唇
-迷奸,
+迷奸
-密穴,
+密穴
-蜜穴,
+蜜穴
-蜜液,
+蜜液
-摸奶,
+摸奶
-摸胸,
+摸胸
-母奸,
+母奸
-奈美,
+奈美
-奶子,
+奶子
-男奴,
+男奴
-内射,
+内射
-嫩逼,
+嫩逼
-嫩女,
+嫩女
-嫩穴,
+嫩穴
-捏弄,
+捏弄
-女优,
+女优
-炮友,
+炮友
-砲友,
+砲友
-喷精,
+喷精
-屁眼,
+屁眼
-前凸后翘,
+前凸后翘
-强jian,
+强jian
-强暴,
+强暴
-强奸处女,
+强奸处女
-情趣用品,
+情趣用品
-情色,
+情色
-拳交,
+拳交
-全裸,
+全裸
-群交,
+群交
-人妻,
+人妻
-人兽,
+人兽
-日逼,
+日逼
-日烂,
+日烂
-肉棒,
+肉棒
-肉逼,
+肉逼
-肉唇,
+肉唇
-肉洞,
+肉洞
-肉缝,
+肉缝
-肉棍,
+肉棍
-肉茎,
+肉茎
-肉具,
+肉具
-揉乳,
+揉乳
-肉穴,
+肉穴
-肉欲,
+肉欲
-乳爆,
+乳爆
-乳房,
+乳房
-乳沟,
+乳沟
-乳交,
+乳交
-乳头,
+乳头
-骚逼,
+骚逼
-骚比,
+骚比
-骚女,
+骚女
-骚水,
+骚水
-骚穴,
+骚穴
-色逼,
+色逼
-色界,
+色界
-色猫,
+色猫
-色盟,
+色盟
-色情网站,
+色情网站
-色区,
+色区
-色色,
+色色
-色诱,
+色诱
-色欲,
+色欲
-色b,
+色b
-少年阿宾,
+少年阿宾
-射爽,
+射爽
-射颜,
+射颜
-食精,
+食精
-释欲,
+释欲
-兽奸,
+兽奸
-兽交,
+兽交
-手淫,
+手淫
-兽欲,
+兽欲
-熟妇,
+熟妇
-熟母,
+熟母
-熟女,
+熟女
-爽片,
+爽片
-双臀,
+双臀
-死逼,
+死逼
-丝袜,
+丝袜
-丝诱,
+丝诱
-松岛枫,
+松岛枫
-酥痒,
+酥痒
-汤加丽,
+汤加丽
-套弄,
+套弄
-体奸,
+体奸
-体位,
+体位
-舔脚,
+舔脚
-舔阴,
+舔阴
-调教,
+调教
-偷欢,
+偷欢
-推油,
+推油
-脱内裤,
+脱内裤
-文做,
+文做
-舞女,
+舞女
-无修正,
+无修正
-吸精,
+吸精
-夏川纯,
+夏川纯
-相奸,
+相奸
-小逼,
+小逼
-校鸡,
+校鸡
-小穴,
+小穴
-小xue,
+小xue
-性感妖娆,
+性感妖娆
-性感诱惑,
+性感诱惑
-性虎,
+性虎
-性饥渴,
+性饥渴
-性技巧,
+性技巧
-性交,
+性交
-性奴,
+性奴
-性虐,
+性虐
-性息,
+性息
-性欲,
+性欲
-胸推,
+胸推
-穴口,
+穴口
-穴图,
+穴图
-亚情,
+亚情
-颜射,
+颜射
-阳具,
+阳具
-杨思敏,
+杨思敏
-要射了,
+要射了
-夜勤病栋,
+夜勤病栋
-一本道,
+一本道
-一夜欢,
+一夜欢
-一夜情,
+一夜情
-一ye情,
+一ye情
-阴部,
+阴部
-淫虫,
+淫虫
-阴唇,
+阴唇
-淫荡,
+淫荡
-阴道,
+阴道
-淫电影,
+淫电影
-阴阜,
+阴阜
-淫妇,
+淫妇
-淫河,
+淫河
-阴核,
+阴核
-阴户,
+阴户
-淫贱,
+淫贱
-淫叫,
+淫叫
-淫教师,
+淫教师
-阴茎,
+阴茎
-阴精,
+阴精
-淫浪,
+淫浪
-淫媚,
+淫媚
-淫糜,
+淫糜
-淫魔,
+淫魔
-淫母,
+淫母
-淫女,
+淫女
-淫虐,
+淫虐
-淫妻,
+淫妻
-淫情,
+淫情
-淫色,
+淫色
-淫声浪语,
+淫声浪语
-淫兽学园,
+淫兽学园
-淫书,
+淫书
-淫术炼金士,
+淫术炼金士
-淫水,
+淫水
-淫娃,
+淫娃
-淫威,
+淫威
-淫亵,
+淫亵
-淫样,
+淫样
-淫液,
+淫液
-淫照,
+淫照
-阴b,
+阴b
-应召,
+应召
-幼交,
+幼交
-欲火,
+欲火
-欲女,
+欲女
-玉乳,
+玉乳
-玉穴,
+玉穴
-援交,
+援交
-原味内衣,
+原味内衣
-援助交际,
+援助交际
-招鸡,
+招鸡
-招妓,
+招妓
-抓胸,
+抓胸
-自慰,
+自慰
-作爱,
+作爱
-a片,
+a片
-fuck,
+fuck
-gay片,
+gay片
-g点,
+g点
-h动画,
+h动画
-h动漫,
+h动漫
-失身粉,
+失身粉
 淫荡自慰器

cmd/server/main.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"github.com/go-playground/validator/v10"
+	"github.com/gofiber/fiber/v2"
+	"log"
+	"os"
+	"sensitive-lexicon/internal/detect"
+	"sensitive-lexicon/internal/lexicon"
+	"strconv"
+	"time"
+)
+
+type User struct {
+	Name       string    `json:"name" validate:"min=5,max=20"`
+	Age        int       `json:"age" validate:"gte=18"`
+	Enrollment time.Time `json:"enrollment" validate:"before_today"`
+	Graduation time.Time `json:"graduation" validate:"gtfield=Enrollment"`
+}
+
+// BeforeToday 验证日期是否在今天之前
+func BeforeToday(fl validator.FieldLevel) bool {
+	fieldTime, ok := fl.Field().Interface().(time.Time)
+	if !ok {
+		return false
+	}
+	return fieldTime.Before(time.Now())
+}
+func main() {
+	lexiconDir := getenv("LEXICON_DIR", "Vocabulary")
+	minNgram := getenvInt("FUZZY_MIN_NGRAM", 2)
+	maxNgram := getenvInt("FUZZY_MAX_NGRAM", 10)
+	maxDistance := getenvInt("FUZZY_MAX_DISTANCE", 1)
+
+	store := lexicon.NewStore()
+	if err := store.LoadFromDir(lexiconDir); err != nil {
+		log.Fatalf("failed to load lexicon: %v", err)
+	}
+
+	service := detect.NewService(store)
+	service.SetFuzzyConfig(detect.FuzzyConfig{MinNgramLen: minNgram, MaxNgramLen: maxNgram, MaxDistance: maxDistance})
+
+	app := fiber.New()
+	app.Get("/health", func(c *fiber.Ctx) error {
+		return c.JSON(fiber.Map{"status": "ok"})
+	})
+
+	app.Post("/detect", func(c *fiber.Ctx) error {
+		var req detect.DetectRequest
+		if err := c.BodyParser(&req); err != nil {
+			return fiber.NewError(fiber.StatusBadRequest, err.Error())
+		}
+		res := service.Detect(req)
+		return c.JSON(res)
+	})
+
+	app.Post("/contains", func(c *fiber.Ctx) error {
+		var req detect.ContainsRequest
+		if err := c.BodyParser(&req); err != nil {
+			return fiber.NewError(fiber.StatusBadRequest, err.Error())
+		}
+		res := service.Contains(req)
+		return c.JSON(res)
+	})
+
+	app.Post("/reload", func(c *fiber.Ctx) error {
+		if err := store.LoadFromDir(lexiconDir); err != nil {
+			return fiber.NewError(fiber.StatusInternalServerError, err.Error())
+		}
+		stats := store.Stats()
+		return c.JSON(stats)
+	})
+
+	port := getenv("PORT", "8080")
+	addr := ":" + port
+	log.Printf("listening on %s", addr)
+	if err := app.Listen(addr); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func getenv(k, def string) string {
+	if v := os.Getenv(k); v != "" {
+		return v
+	}
+	return def
+}
+
+func getenvInt(k string, def int) int {
+	if v := os.Getenv(k); v != "" {
+		if i, err := strconv.Atoi(v); err == nil {
+			return i
+		}
+	}
+	return def
+}

go.mod

@@ -0,0 +1,30 @@
+module sensitive-lexicon
+
+go 1.22
+
+require (
+	github.com/go-playground/validator/v10 v10.27.0
+	github.com/gofiber/fiber/v2 v2.52.5
+	github.com/ozeidan/fuzzy-patricia v3.0.0+incompatible
+)
+
+require (
+	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/google/uuid v1.5.0 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/fasthttp v1.51.0 // indirect
+	github.com/valyala/tcplisten v1.0.0 // indirect
+	golang.org/x/crypto v0.33.0 // indirect
+	golang.org/x/net v0.34.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
+	golang.org/x/text v0.22.0 // indirect
+)

go.sum

@@ -0,0 +1,55 @@
+github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
+github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
+github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
+github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
+github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4=
+github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
+github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
+github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
+github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
+github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/ozeidan/fuzzy-patricia v3.0.0+incompatible h1:Pl61eMyfJqgY/wytiI4vamqPYribq6d8VxeP1CNyg9M=
+github.com/ozeidan/fuzzy-patricia v3.0.0+incompatible/go.mod h1:zgvuCcYS7wB7fVCGblsaFFmEe8+aAH13dTYm8FbrpsM=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
+github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
+github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
+github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
+golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
+golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
+golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
+golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/detect/service.go

@@ -0,0 +1,144 @@
+package detect
+
+import (
+	"sort"
+	"strings"
+	"unicode/utf8"
+
+	"sensitive-lexicon/internal/lexicon"
+)
+
+type FuzzyConfig struct {
+	MinNgramLen int
+	MaxNgramLen int
+	MaxDistance int
+}
+
+type DetectRequest struct {
+	Text string `json:"text"`
+	// If true, enable fuzzy detection on n-grams within the text
+	EnableFuzzy bool `json:"enable_fuzzy"`
+}
+
+type Match struct {
+	Word     string `json:"word"`
+	Type     string `json:"type"` // substring | fuzzy
+	Distance int    `json:"distance,omitempty"`
+}
+
+type DetectResponse struct {
+	Hits []Match `json:"hits"`
+}
+
+type ContainsRequest struct {
+	Text string `json:"text"`
+}
+
+type ContainsResponse struct {
+	Contains bool   `json:"contains"`
+	Word     string `json:"word,omitempty"`
+}
+
+type Service struct {
+	store    *lexicon.Store
+	fuzzyCfg FuzzyConfig
+}
+
+func NewService(store *lexicon.Store) *Service {
+	return &Service{store: store, fuzzyCfg: FuzzyConfig{MinNgramLen: 2, MaxNgramLen: 10, MaxDistance: 1}}
+}
+
+func (s *Service) SetFuzzyConfig(cfg FuzzyConfig) {
+	s.fuzzyCfg = cfg
+}
+
+func (s *Service) Detect(req DetectRequest) DetectResponse {
+	text := strings.TrimSpace(req.Text)
+	if text == "" {
+		return DetectResponse{}
+	}
+
+	unique := make(map[string]Match)
+
+	// Substring hits: for each codepoint window from input, find lexicon entries containing it
+	s.store.ForEachSubstringMatch(text, func(word string) bool {
+		unique[word] = Match{Word: word, Type: "substring"}
+		return true
+	})
+
+	if req.EnableFuzzy {
+		for _, token := range generateNgrams(text, s.fuzzyCfg.MinNgramLen, s.fuzzyCfg.MaxNgramLen) {
+			s.store.ForEachFuzzyMatch(token, s.fuzzyCfg.MaxDistance, func(word string, d int) bool {
+				if old, ok := unique[word]; ok {
+					if old.Type == "substring" && d == 0 {
+						return true
+					}
+				}
+				unique[word] = Match{Word: word, Type: ternary(d == 0, "substring", "fuzzy"), Distance: d}
+				return true
+			})
+		}
+	}
+
+	res := DetectResponse{Hits: make([]Match, 0, len(unique))}
+	for _, v := range unique {
+		res.Hits = append(res.Hits, v)
+	}
+	sort.Slice(res.Hits, func(i, j int) bool {
+		if res.Hits[i].Type == res.Hits[j].Type {
+			if res.Hits[i].Distance == res.Hits[j].Distance {
+				return res.Hits[i].Word < res.Hits[j].Word
+			}
+			return res.Hits[i].Distance < res.Hits[j].Distance
+		}
+		return res.Hits[i].Type < res.Hits[j].Type
+	})
+	return res
+}
+
+func (s *Service) Contains(req ContainsRequest) ContainsResponse {
+	ok, w := s.store.HasAnyInText(strings.TrimSpace(req.Text))
+	return ContainsResponse{Contains: ok, Word: w}
+}
+
+func ternary[T any](cond bool, a, b T) T {
+	if cond {
+		return a
+	}
+	return b
+}
+
+func generateNgrams(text string, minLen, maxLen int) []string {
+	if minLen < 1 {
+		minLen = 1
+	}
+	if maxLen < minLen {
+		maxLen = minLen
+	}
+	// Work on rune boundaries for CJK safety
+	runes := []rune(text)
+	n := len(runes)
+	var out []string
+	for i := 0; i < n; i++ {
+		for l := minLen; l <= maxLen && i+l <= n; l++ {
+			out = append(out, string(runes[i:i+l]))
+		}
+	}
+	return dedupStrings(out)
+}
+
+func dedupStrings(in []string) []string {
+	seen := make(map[string]struct{}, len(in))
+	out := make([]string, 0, len(in))
+	for _, s := range in {
+		if _, ok := seen[s]; ok {
+			continue
+		}
+		seen[s] = struct{}{}
+		out = append(out, s)
+	}
+	return out
+}
+
+// Guard for unused import warning if utf8 not referenced elsewhere
+var _ = utf8.RuneCountInString

internal/lexicon/store.go

@@ -0,0 +1,141 @@
+package lexicon
+
+import (
+	"bufio"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/ozeidan/fuzzy-patricia/patricia"
+)
+
+// Store holds the trie and statistics for the loaded lexicon.
+type Store struct {
+	mu   sync.RWMutex
+	trie *patricia.Trie
+	cnt  int
+}
+
+func NewStore() *Store {
+	return &Store{trie: patricia.NewTrie()}
+}
+
+// LoadFromDir loads all .txt files from dir into the trie.
+func (s *Store) LoadFromDir(dir string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	newTrie := patricia.NewTrie()
+	count := 0
+
+	walkErr := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		if strings.ToLower(filepath.Ext(info.Name())) != ".txt" {
+			return nil
+		}
+		f, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		scanner := bufio.NewScanner(f)
+		// Increase buffer for long lines
+		buf := make([]byte, 0, 1024*64)
+		scanner.Buffer(buf, 1024*1024)
+		for scanner.Scan() {
+			w := strings.TrimSpace(scanner.Text())
+			if w == "" || strings.HasPrefix(w, "#") {
+				continue
+			}
+			newTrie.Insert(patricia.Prefix(w), struct{}{})
+			count++
+		}
+		return scanner.Err()
+	})
+	if walkErr != nil {
+		return walkErr
+	}
+	if count == 0 {
+		return errors.New("no entries loaded")
+	}
+	// Swap in
+	s.trie = newTrie
+	s.cnt = count
+	return nil
+}
+
+func (s *Store) Stats() map[string]interface{} {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	return map[string]interface{}{
+		"count": s.cnt,
+	}
+}
+
+// ForEachSubstringMatch visits any keys that contain the given substring.
+// It uses the library's substring search.
+func (s *Store) ForEachSubstringMatch(query string, visit func(word string) bool) {
+	s.mu.RLock()
+	tr := s.trie
+	s.mu.RUnlock()
+	if tr == nil || query == "" {
+		return
+	}
+	// second argument is caseSensitive; we use false by default
+	tr.VisitSubstring(patricia.Prefix(query), false, func(prefix patricia.Prefix, _ patricia.Item) error {
+		// The library does not expose a public stop error in all versions; ignore early stop
+		_ = visit(string(prefix))
+		return nil
+	})
+}
+
+// ForEachFuzzyMatch visits keys with fuzzy distance within maxDistance to query.
+func (s *Store) ForEachFuzzyMatch(query string, maxDistance int, visit func(word string, distance int) bool) {
+	s.mu.RLock()
+	tr := s.trie
+	s.mu.RUnlock()
+	if tr == nil || query == "" {
+		return
+	}
+	// signature in current lib: VisitFuzzy(prefix, caseSensitive bool, visitor)
+	tr.VisitFuzzy(patricia.Prefix(query), false, func(prefix patricia.Prefix, _ patricia.Item, dist int) error {
+		if dist <= maxDistance {
+			_ = visit(string(prefix), dist)
+		}
+		return nil
+	})
+}
+
+// HasAnyInText returns true if any lexicon word is a substring of the given text.
+// It scans each rune offset and visits prefixes against the trie.
+func (s *Store) HasAnyInText(text string) (bool, string) {
+	s.mu.RLock()
+	tr := s.trie
+	s.mu.RUnlock()
+	if tr == nil || text == "" {
+		return false, ""
+	}
+	runes := []rune(text)
+	n := len(runes)
+	for i := 0; i < n; i++ {
+		suffix := string(runes[i:])
+		foundWord := ""
+		tr.VisitPrefixes(patricia.Prefix(suffix), false, func(prefix patricia.Prefix, _ patricia.Item) error {
+			if foundWord == "" {
+				foundWord = string(prefix)
+			}
+			return nil
+		})
+		if foundWord != "" {
+			return true, foundWord
+		}
+	}
+	return false, ""
+}