opt:语音控制功能-第一次联调测试修改

nladmin-system/nlsso-server/src/main/java/org/nl/wms/basedata_manage/service/dao/mapper/StructattrMapper.xml

@@ -35,6 +35,9 @@
             <if test="pcsn != null and pcsn != ''">
                 AND gro.pcsn = #{pcsn}
             </if>
+            <if test="struct_code != null and struct_code != ''">
+                AND ivt.struct_code = #{struct_code}
+            </if>
             <if test="stor_code != null and stor_code != ''">
                 AND ivt.stor_code = #{stor_code}
             </if>

nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/controller/speech/SysSpeechController.java

@@ -16,7 +16,7 @@ import java.util.Map;
  */
 @Slf4j
 @RestController
-@RequestMapping("/speech")
+@RequestMapping("/api/speech")
 public class SysSpeechController {
 
     @Resource
@@ -56,7 +56,7 @@ public class SysSpeechController {
      */
     @PostMapping("/confirm")
     public ResponseEntity<Object> confirm(@RequestBody Map<String, Object> body){
-        sysSpeechService.confirm(body.getOrDefault("text","").toString());
+        Map<String, Object> res = sysSpeechService.confirm(body.getOrDefault("text", "").toString());
-        return new ResponseEntity<>(HttpStatus.OK);
+        return ResponseEntity.ok(res);
     }
 }

nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/controller/speech/SysSpeechWebSocketEndpoint.java

@@ -56,46 +56,42 @@ public class SysSpeechWebSocketEndpoint {
         if ("start".equals(type)) {
             //TODO 健康检查
             asrTtsClient.health().thenAccept(result -> {
+                log.info("健康检查结果: {}", result);
                 Integer resCode = JSON.parseObject(result.toString()).getInteger("code");
                 if (resCode != 0) {
-                    sendError("语音控制服务异常");
+                    sendTextBySys("语音控制服务异常");
                 }
             }).exceptionally(ex -> {
                 log.error("健康检查异常: {}", session.getId(), ex);
-                sendError("语音控制服务异常");
+                sendTextBySys("语音控制服务异常");
                 return null;
             });
         } else if ("end".equals(type)) {
             // 录音结束，将 PCM 数据发送给第三方 ASR
             byte[] pcmData = ctx.audioBuffer.toByteArray();
-            //测试
-//            String resp = JSONObject.toJSONString(MapOf.of("type", "transcription", "text", "识别结果"));
-//            String resp1 = JSONObject.toJSONString(MapOf.of("type", "response", "text", "系统回复消息"));
-//            session.getBasicRemote().sendText(resp);
-//            session.getBasicRemote().sendText(resp1);
-            //测试结束
 
             // 异步调用第三方服务，避免阻塞 WebSocket 线程
             asrTtsClient.recognizeAsync(pcmData, ctx.sampleRate, ctx.channels, ctx.bitDepth)
                     .thenAccept(result -> {
                         JSONObject jsonObject = JSON.parseObject(result.toString(), JSONObject.class);
+                        log.info("ASR 输出参数为：-------------------" + jsonObject.toString());
                         Integer code = jsonObject.getInteger("code");
                         if (code == null || code != 0) {
                             String msg = jsonObject.getString("msg");
-                            sendError(msg != null ? msg : "未知错误");
+                            sendTextBySys(msg != null ? msg : "未知错误");
                             return;
                         }
                         JSONObject data = jsonObject.getJSONObject("data");
                         if (data != null) {
                             String text = data.getString("text");
-                            sendText(text);
+                            sendTextByUser(text);
                         } else {
-                            sendError("语音转文字发生错误");
+                            sendTextBySys("语音转文字发生错误");
                         }
                     })
                     .exceptionally(ex -> {
                         log.error("ASR 异常: {}", session.getId(), ex);
-                        sendError("ASR 异常");
+                        sendTextByUser("ASR 异常");
                         return null;
                     });
             ctx.audioBuffer.reset();
@@ -144,11 +140,11 @@ public class SysSpeechWebSocketEndpoint {
         }
     }
 
-    private void sendText(String message) {
+    private void sendTextByUser(String message) {
         try {
             Map<String, Object> map = new HashMap<>();
-            map.put("type", "result");
+            map.put("type", "user");
-            map.put("text", message);
+            map.put("message", message);
             // 结果通过 WebSocket 返回给前端
             String response = objectMapper.writeValueAsString(map);
             session.getBasicRemote().sendText(response);
@@ -157,10 +153,10 @@ public class SysSpeechWebSocketEndpoint {
         }
     }
 
-    private void sendError(String message) {
+    private void sendTextBySys(String message) {
         try {
             Map<String, String> map = new HashMap<>();
-            map.put("type", "error");
+            map.put("type", "system");
             map.put("message", message);
             session.getBasicRemote().sendText(
                     objectMapper.writeValueAsString(map));

nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/speech/VoiceCallbackService.java

@@ -85,8 +85,10 @@ public class VoiceCallbackService {
 
     @Transactional
     public JSONObject handleInboundTask(InboundTaskRequest inboundTaskRequest){
+//        SchBasePoint one = iSchBasePointService.getOne(new LambdaQueryWrapper<SchBasePoint>()
+//                .eq(SchBasePoint::getPoint_code, inboundTaskRequest.getLocationFrom()));
         SchBasePoint one = iSchBasePointService.getOne(new LambdaQueryWrapper<SchBasePoint>()
-                .eq(SchBasePoint::getPoint_code, inboundTaskRequest.getLocationFrom()));
+                .eq(SchBasePoint::getPoint_code, "RKD01"));
         //分配仓位
         MdMeMaterialbase meMaterialbase = iMdMeMaterialbaseService.getByCode(inboundTaskRequest.getMaterial());
 
@@ -114,8 +116,8 @@ public class VoiceCallbackService {
         List<Structattr> structattrs = iStructattrService.inBoundSectDiv(
                 StrategyStructParam.builder()
                         .ioType(StatusEnum.STRATEGY_TYPE.code("入库"))
-                        .sect_code("BG01")
+                        .sect_code("YL01")
-                        .stor_code("BGK")
+                        .stor_code("YL")
                         .storagevehicle_code(one.getVehicle_code())
                         .strategyMaters(materss)
                         .build());
@@ -124,23 +126,28 @@ public class VoiceCallbackService {
         }
         Structattr attrDao = structattrs.get(0);
         //确定起点
-        SchBasePoint schBasePoint = iSchBasePointService.getOne(new LambdaQueryWrapper<SchBasePoint>()
+//        SchBasePoint schBasePoint = iSchBasePointService.getOne(new LambdaQueryWrapper<SchBasePoint>()
-                .eq(SchBasePoint::getPoint_code, inboundTaskRequest.getLocationFrom()));
+//                .eq(SchBasePoint::getPoint_code, inboundTaskRequest.getLocationFrom()));
-        if (ObjectUtil.isEmpty(schBasePoint)) {
+//        if (ObjectUtil.isEmpty(schBasePoint)) {
-            throw new BadRequestException("未找到载具所在的点位信息,请检查");
+//            throw new BadRequestException("未找到载具所在的点位信息,请检查");
-        }
+//        }
         JSONObject whereJson = new JSONObject();
-        whereJson.put("point_code1", schBasePoint.getPoint_code());
+//        whereJson.put("point_code1", "RKD01");
+//        whereJson.put("point_code1", schBasePoint.getPoint_code());
         //确定终点
-        whereJson.put("point_code2", attrDao.getStruct_code());
+//        whereJson.put("point_code2", attrDao.getStruct_code());
         whereJson.put("vehicle_code", one.getVehicle_code());
+        whereJson.put("task_type", "STInTask");
+        whereJson.put("TaskCode", CodeUtil.getNewCode("TASK_CODE"));
+        whereJson.put("PickingLocation", "RKD01");
+        whereJson.put("PlacedLocation", attrDao.getStruct_code());
         //创建任务
-        String taskId = applyTaskMap.get("VehicleInTask").create(whereJson);
+        String taskId = applyTaskMap.get("STInTask").create(whereJson);
         // 更新起点绑定id
         iSchBasePointService.update(
                 new UpdateWrapper<SchBasePoint>().lambda()
                         .set(SchBasePoint::getIos_id, null)
-                        .eq(SchBasePoint::getPoint_code, schBasePoint.getPoint_code())
+                        .eq(SchBasePoint::getPoint_code, "RKD01")
         );
         // 更新终点锁定状态
         JSONObject lock_map = new JSONObject();
@@ -159,8 +166,8 @@ public class VoiceCallbackService {
         JSONObject whereJson = new JSONObject();
         whereJson.put("material_id", meMaterialbase.getMaterial_id());
         whereJson.put("material_code", meMaterialbase.getMaterial_code());
-        whereJson.put("stor_code", "BGK");
+        whereJson.put("stor_code", "YL");
-        whereJson.put("sect_code", "BG01");
+        whereJson.put("sect_code", "YL01");
         whereJson.put("siteCode", "CKD01");
         StrategyMater mater = new StrategyMater();
         mater.setQty(BigDecimal.valueOf(outboundTaskRequest.getQty()));
@@ -214,19 +221,33 @@ public class VoiceCallbackService {
         return null;
     };
     public JSONObject handleMoveTask(MoveTaskRequest moveTaskRequest){
-        String taskId = applyTaskMap.get("StructMoveTask").create((JSONObject) JSONObject.toJSON(moveTaskRequest));
+        Structattr attrDao = iStructattrService.findByCode(moveTaskRequest.getLocationFrom());
+        JSONObject taskForm = new JSONObject();
+        taskForm.put("point_code1", moveTaskRequest.getLocationFrom());
+        taskForm.put("point_code", moveTaskRequest.getLocationTo());
+        taskForm.put("config_code", IOSConstant.MOVE_CONFIG_TASK);
+        taskForm.put("vehicle_code", attrDao.getStoragevehicle_code());
+        String taskId = applyTaskMap.get("MoveTask").create((JSONObject) JSONObject.toJSON(taskForm));
         return null;
     };
 
     public JSONObject handleQueryTask(QueryBoundRequest queryBoundRequest){
         List<StructattrVechielDto> structCode = iStructattrService.collectVechicle(MapOf.of("struct_code", queryBoundRequest.getLocation(), "search", queryBoundRequest.getMaterial()));
+        Map<String, BigDecimal> collect = structCode.stream()
+                .collect(Collectors.groupingBy(StructattrVechielDto::getMaterial_code,
+                        Collectors.reducing(BigDecimal.ZERO, StructattrVechielDto::getQty, BigDecimal::add)));
         StringBuffer stringBuffer = new StringBuffer();
         stringBuffer.append("当前仓库物料有物料");
-        for (StructattrVechielDto vechielDtos : structCode) {
+        collect.entrySet().forEach(r -> {
-            stringBuffer.append(vechielDtos.getMaterial_name()).append("共计")
+            stringBuffer.append(r.getKey()).append("共计")
-                    .append(vechielDtos.getQty())
+                    .append(r.getValue())
                     .append("个");
-        }
+        });
+//        for (StructattrVechielDto vechielDtos : structCode) {
+//            stringBuffer.append(vechielDtos.getMaterial_name()).append("共计")
+//                    .append(vechielDtos.getQty())
+//                    .append("个");
+//        }
         String text = stringBuffer.toString();
         try {
 //            voiceService.speechTts(text);

nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/speech/api/AsrTtsClient.java

@@ -18,6 +18,7 @@ public class AsrTtsClient {
 
     @Value("${vc.base-url:http://127.0.0.1:5000}")
     private String vcBaseUrl;
+
     /**
      * 健康检查
      */
@@ -29,12 +30,14 @@ public class AsrTtsClient {
         });
 
     }
+
     /**
      * 语音转文字
      */
     public static void asr() {
 
     }
+
     /**
      * 文字转语音
      */
@@ -55,11 +58,11 @@ public class AsrTtsClient {
         });
 
     }
+
     /**
      * 用户确认文字
      */
-    public CompletableFuture<Object> confirm(String  text, boolean confirmed) {
+    public String confirm(String text, boolean confirmed) {
-        return CompletableFuture.supplyAsync(() -> {
         Map<String, Object> params = new HashMap<>();
         params.put("text", text);
         params.put("confirmed", confirmed);
@@ -68,7 +71,6 @@ public class AsrTtsClient {
                 .contentType("application/json")
                 .execute()
                 .body();
-        });
 
     }
 
@@ -76,7 +78,8 @@ public class AsrTtsClient {
 
         return CompletableFuture.supplyAsync(() -> {
             return HttpRequest.post(vcBaseUrl + "/api/speech/asr")
-                    .form("audio", pcmData)
+                    .form("audio", pcmData, "audio.pcm")
+//                    .body(pcmData)
                     .contentType("multipart/form-data")
                     .execute()
                     .body();

nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/speech/impl/SysSpeechServiceImpl.java

@@ -1,5 +1,6 @@
 package org.nl.wms.system_manage.service.speech.impl;
 
+import com.alibaba.fastjson.JSONObject;
 import lombok.extern.slf4j.Slf4j;
 import org.nl.common.utils.MapOf;
 import org.nl.wms.system_manage.service.speech.SysSpeechService;
@@ -39,18 +40,20 @@ public class SysSpeechServiceImpl implements SysSpeechService {
 
     @Override
     public Map<String, Object> confirm(String text) {
-        if ("".equals( text)){
+        if ("".equals(text)) {
             throw new RuntimeException("确认文本为空");
         }
-        asrTtsClient.confirm(text, true)
+        String resJson = asrTtsClient.confirm(text, true);
-                .thenAccept(result -> {
+        log.info("语音控制：用户确认结果: {}", resJson);
-                    log.info("语音控制：用户确认结果: {}", result);
+        //TODO
-                }).exceptionally(ex -> {
+        JSONObject jsonObject = JSONObject.parseObject(resJson);
-                    log.error("语音控制：用户确认异常: {}", ex);
+        JSONObject data = jsonObject.getJSONObject("data");
-                    return null;
+        if (data != null) {
-                });
+            String resText = data.getString("text");
+            return MapOf.of("text", resText);
+        }
         byte[] audio = asrTtsClient.tts(text).join();
         //TODO 音频返回前端播放处理
-        return Collections.emptyMap();
+        return MapOf.of("text", text);
     }
 }

nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml

@@ -9,10 +9,11 @@ spring:
     druid:
       db-type: com.alibaba.druid.pool.DruidDataSource
       driverClassName: com.mysql.cj.jdbc.Driver
-      url: jdbc:mysql://${DB_HOST:192.168.81.251}:${DB_PORT:3306}/${DB_NAME:wms_standardv2}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true&useSSL=false
+#      url: jdbc:mysql://${DB_HOST:192.168.81.251}:${DB_PORT:3306}/${DB_NAME:wms_standardv2}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true&useSSL=false
-      # url: jdbc:mysql://${DB_HOST:localhost}:${DB_PORT:3306}/${DB_NAME:wms_oulun}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true&useSSL=false
+      url: jdbc:mysql://${DB_HOST:localhost}:${DB_PORT:3306}/${DB_NAME:wms_standardv2}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true&useSSL=false
       username: ${DB_USER:root}
-      password: ${DB_PWD:P@ssw0rd.}
+      password: ${DB_PWD:123456}
+#      password: ${DB_PWD:P@ssw0rd.}
       # 初始连接数
       initial-size: 15
       # 最小连接数
@@ -137,3 +138,5 @@ sa-token:
 lucene:
   index:
     path: D:\lms\lucene\index
+vc:
+  base-url: http://192.168.20.156:5000

nladmin-ui/src/components/VoiceInputDialog/index.vue

@@ -133,10 +133,18 @@
           <span>正在录音... {{ recordingDuration }}"</span>
         </div>
 
+        <!-- 识别中提示 -->
+        <div v-if="isTranscribing" class="transcribing-hint">
+          <div class="transcribing-spinner">
+            <i class="el-icon-loading" />
+          </div>
+          <span>正在识别录音内容...</span>
+        </div>
+
         <!-- 录音按钮 -->
         <div
           class="record-button"
-          :class="{ 'recording': isRecording, 'disabled': !isConnected || isProcessing }"
+          :class="{ 'recording': isRecording, 'disabled': !isConnected || isProcessing || isTranscribing }"
           @mousedown="startRecording"
           @mouseup="stopRecording"
           @mouseleave="isRecording && stopRecording()"
@@ -240,7 +248,7 @@ export default {
       this.isConnected = false
     },
 
-    // 处理WebSocket消息
+    // 处理WebSocket消息 - 只处理user和system两种类型
     handleWebSocketMessage(data) {
       // 判断数据类型
       if (data instanceof ArrayBuffer || data instanceof Blob) {
@@ -252,14 +260,21 @@ export default {
           const message = JSON.parse(data)
           console.log('收到WebSocket消息:', message)
 
-          if (message.type === 'error') {
+          // 只处理user和system两种类型
-            this.handleError(message.message || '识别失败')
+          if (message.type === 'user') {
-          } else if (message.type === 'transcription') {
+            // 用户识别结果消息
-            // 语音识别结果
+            this.handleUserMessage(message)
-            this.handleTranscriptionResult(message)
+          } else if (message.type === 'system') {
-          } else if (message.type === 'response') {
+            // 系统回复消息
-            // 系统文本回复
+            this.handleSystemMessage(message)
-            this.handleSystemTextResponse(message)
+          } else if (message.type === 'error') {
+            // 错误消息作为系统消息显示
+            this.addMessage({
+              role: 'system',
+              type: 'text',
+              content: message.message || '处理失败',
+              timestamp: Date.now()
+            })
           }
         } catch (e) {
           console.error('解析WebSocket消息失败:', e)
@@ -267,9 +282,9 @@ export default {
       }
     },
 
-    // 处理语音识别结果
+    // 处理用户消息（语音识别结果）
-    handleTranscriptionResult(message) {
+    handleUserMessage(message) {
-      const text = message.text || message.transcript || ''
+      const text = message.message
       if (!text) return
 
       this.isTranscribing = false
@@ -284,6 +299,19 @@ export default {
       })
     },
 
+    // 处理系统消息
+    handleSystemMessage(message) {
+      this.isProcessing = false
+
+      // 添加系统消息
+      this.addMessage({
+        role: 'system',
+        type: 'text',
+        content: message.message,
+        timestamp: Date.now()
+      })
+    },
+
     // 确认消息 - 用户点击确认按钮后执行
     async confirmMessage(message) {
       if (!message || !message.content) return
@@ -294,14 +322,16 @@ export default {
 
       // 执行confirm函数
       try {
-        const res = await confirm(message.content)
+        const res = await confirm({
+          text: message.content
+        })
         this.isProcessing = false
 
         // 添加系统回复消息
         this.addMessage({
           role: 'system',
           type: 'text',
-          content: res.data || '收到您的消息',
+          content: res.text,
           timestamp: Date.now()
         })
       } catch (error) {
@@ -328,18 +358,7 @@ export default {
       this.scrollToBottom()
     },
 
-    // 处理系统文本回复
+    // 处理系统语音回复（二进制数据）
-    handleSystemTextResponse(message) {
-      this.isProcessing = false
-      this.addMessage({
-        role: 'system',
-        type: 'text',
-        content: message.text || message.content || '收到您的消息',
-        timestamp: Date.now()
-      })
-    },
-
-    // 处理系统语音回复
     async handleSystemVoiceResponse(audioData) {
       try {
         // 创建音频URL
@@ -360,7 +379,7 @@ export default {
           role: 'system',
           type: 'voice',
           audioUrl: audioUrl,
-          duration: '3"', // 这里应该根据实际音频时长计算
+          duration: '3"',
           timestamp: Date.now()
         }
         this.addMessage(message)
@@ -372,27 +391,7 @@ export default {
       }
     },
 
-    // 处理错误
-    handleError(message) {
-      this.isProcessing = false
-      this.isTranscribing = false
-      this.addMessage({
-        role: 'system',
-        type: 'text',
-        content: message || '服务出现错误',
-        timestamp: Date.now()
-      })
 
-      // 更新最后一条用户消息状态
-      // const lastUserMessage = this.messages.filter(m => m.role === 'user').pop()
-      // if (lastUserMessage) {
-      //   lastUserMessage.isTranscribing = false
-      //   lastUserMessage.content = message
-      //   lastUserMessage.type = 'text'
-      // }
-      //
-      // this.$message.error(message)
-    },
 
     // 添加消息
     addMessage(message) {
@@ -1030,6 +1029,26 @@ export default {
         }
       }
 
+      .transcribing-hint {
+        display: flex;
+        align-items: center;
+        justify-content: center;
+        margin-bottom: 10px;
+        color: #409EFF;
+        font-size: 14px;
+
+        .transcribing-spinner {
+          display: flex;
+          align-items: center;
+          margin-right: 8px;
+
+          i {
+            font-size: 18px;
+            animation: rotating 1s linear infinite;
+          }
+        }
+      }
+
       .record-button {
       display: flex;
       align-items: center;