root/WmsStandard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

opt:语音控制功能-第一次联调测试修改

Browse Source
This commit is contained in:
zhaoyf
2026-04-22 10:47:24 +08:00
parent 5d83cceca5
commit bced456945
8 changed files with 178 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
nladmin-system/nlsso-server/src/main/java/org/nl/wms/basedata_manage/service/dao/mapper/StructattrMapper.xml
View File

@@ -35,6 +35,9 @@
<if test="pcsn != null and pcsn != ''">
AND gro.pcsn = #{pcsn}
</if>
<if test="struct_code != null and struct_code != ''">
AND ivt.struct_code = #{struct_code}
</if>
<if test="stor_code != null and stor_code != ''">
AND ivt.stor_code = #{stor_code}
</if>

6
nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/controller/speech/SysSpeechController.java
View File

@@ -16,7 +16,7 @@ import java.util.Map;
*/
@Slf4j
@RestController
@RequestMapping("/speech")
@RequestMapping("/api/speech")
public class SysSpeechController {
@Resource
@@ -56,7 +56,7 @@ public class SysSpeechController {
*/
@PostMapping("/confirm")
public ResponseEntity<Object> confirm(@RequestBody Map<String, Object> body){
sysSpeechService.confirm(body.getOrDefault("text","").toString());
return new ResponseEntity<>(HttpStatus.OK);
Map<String, Object> res = sysSpeechService.confirm(body.getOrDefault("text", "").toString());
return ResponseEntity.ok(res);
}
}

30
nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/controller/speech/SysSpeechWebSocketEndpoint.java
View File

@@ -56,46 +56,42 @@ public class SysSpeechWebSocketEndpoint {
if ("start".equals(type)) {
//TODO 健康检查
asrTtsClient.health().thenAccept(result -> {
log.info("健康检查结果: {}", result);
Integer resCode = JSON.parseObject(result.toString()).getInteger("code");
if (resCode != 0) {
sendError("语音控制服务异常");
sendTextBySys("语音控制服务异常");
}
}).exceptionally(ex -> {
log.error("健康检查异常: {}", session.getId(), ex);
sendError("语音控制服务异常");
sendTextBySys("语音控制服务异常");
return null;
});
} else if ("end".equals(type)) {
// 录音结束，将 PCM 数据发送给第三方 ASR
byte[] pcmData = ctx.audioBuffer.toByteArray();
//测试
// String resp = JSONObject.toJSONString(MapOf.of("type", "transcription", "text", "识别结果"));
// String resp1 = JSONObject.toJSONString(MapOf.of("type", "response", "text", "系统回复消息"));
// session.getBasicRemote().sendText(resp);
// session.getBasicRemote().sendText(resp1);
//测试结束
// 异步调用第三方服务，避免阻塞 WebSocket 线程
asrTtsClient.recognizeAsync(pcmData, ctx.sampleRate, ctx.channels, ctx.bitDepth)
.thenAccept(result -> {
JSONObject jsonObject = JSON.parseObject(result.toString(), JSONObject.class);
log.info("ASR 输出参数为：-------------------" + jsonObject.toString());
Integer code = jsonObject.getInteger("code");
if (code == null || code != 0) {
String msg = jsonObject.getString("msg");
sendError(msg != null ? msg : "未知错误");
sendTextBySys(msg != null ? msg : "未知错误");
return;
}
JSONObject data = jsonObject.getJSONObject("data");
if (data != null) {
String text = data.getString("text");
sendText(text);
sendTextByUser(text);
} else {
sendError("语音转文字发生错误");
sendTextBySys("语音转文字发生错误");
}
})
.exceptionally(ex -> {
log.error("ASR 异常: {}", session.getId(), ex);
sendError("ASR 异常");
sendTextByUser("ASR 异常");
return null;
});
ctx.audioBuffer.reset();
@@ -144,11 +140,11 @@ public class SysSpeechWebSocketEndpoint {
}
}
private void sendText(String message) {
private void sendTextByUser(String message) {
try {
Map<String, Object> map = new HashMap<>();
map.put("type", "result");
map.put("text", message);
map.put("type", "user");
map.put("message", message);
// 结果通过 WebSocket 返回给前端
String response = objectMapper.writeValueAsString(map);
session.getBasicRemote().sendText(response);
@@ -157,10 +153,10 @@ public class SysSpeechWebSocketEndpoint {
}
}
private void sendError(String message) {
private void sendTextBySys(String message) {
try {
Map<String, String> map = new HashMap<>();
map.put("type", "error");
map.put("type", "system");
map.put("message", message);
session.getBasicRemote().sendText(
objectMapper.writeValueAsString(map));

59
nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/speech/VoiceCallbackService.java
View File

@@ -85,8 +85,10 @@ public class VoiceCallbackService {
@Transactional
public JSONObject handleInboundTask(InboundTaskRequest inboundTaskRequest){
// SchBasePoint one = iSchBasePointService.getOne(new LambdaQueryWrapper<SchBasePoint>()
// .eq(SchBasePoint::getPoint_code, inboundTaskRequest.getLocationFrom()));
SchBasePoint one = iSchBasePointService.getOne(new LambdaQueryWrapper<SchBasePoint>()
.eq(SchBasePoint::getPoint_code, inboundTaskRequest.getLocationFrom()));
.eq(SchBasePoint::getPoint_code, "RKD01"));
//分配仓位
MdMeMaterialbase meMaterialbase = iMdMeMaterialbaseService.getByCode(inboundTaskRequest.getMaterial());
@@ -114,8 +116,8 @@ public class VoiceCallbackService {
List<Structattr> structattrs = iStructattrService.inBoundSectDiv(
StrategyStructParam.builder()
.ioType(StatusEnum.STRATEGY_TYPE.code("入库"))
.sect_code("BG01")
.stor_code("BGK")
.sect_code("YL01")
.stor_code("YL")
.storagevehicle_code(one.getVehicle_code())
.strategyMaters(materss)
.build());
@@ -124,23 +126,28 @@ public class VoiceCallbackService {
}
Structattr attrDao = structattrs.get(0);
//确定起点
SchBasePoint schBasePoint = iSchBasePointService.getOne(new LambdaQueryWrapper<SchBasePoint>()
.eq(SchBasePoint::getPoint_code, inboundTaskRequest.getLocationFrom()));
if (ObjectUtil.isEmpty(schBasePoint)) {
throw new BadRequestException("未找到载具所在的点位信息,请检查");
}
// SchBasePoint schBasePoint = iSchBasePointService.getOne(new LambdaQueryWrapper<SchBasePoint>()
// .eq(SchBasePoint::getPoint_code, inboundTaskRequest.getLocationFrom()));
// if (ObjectUtil.isEmpty(schBasePoint)) {
// throw new BadRequestException("未找到载具所在的点位信息,请检查");
// }
JSONObject whereJson = new JSONObject();
whereJson.put("point_code1", schBasePoint.getPoint_code());
// whereJson.put("point_code1", "RKD01");
// whereJson.put("point_code1", schBasePoint.getPoint_code());
//确定终点
whereJson.put("point_code2", attrDao.getStruct_code());
// whereJson.put("point_code2", attrDao.getStruct_code());
whereJson.put("vehicle_code", one.getVehicle_code());
whereJson.put("task_type", "STInTask");
whereJson.put("TaskCode", CodeUtil.getNewCode("TASK_CODE"));
whereJson.put("PickingLocation", "RKD01");
whereJson.put("PlacedLocation", attrDao.getStruct_code());
//创建任务
String taskId = applyTaskMap.get("VehicleInTask").create(whereJson);
String taskId = applyTaskMap.get("STInTask").create(whereJson);
// 更新起点绑定id
iSchBasePointService.update(
new UpdateWrapper<SchBasePoint>().lambda()
.set(SchBasePoint::getIos_id, null)
.eq(SchBasePoint::getPoint_code, schBasePoint.getPoint_code())
.eq(SchBasePoint::getPoint_code, "RKD01")
);
// 更新终点锁定状态
JSONObject lock_map = new JSONObject();
@@ -159,8 +166,8 @@ public class VoiceCallbackService {
JSONObject whereJson = new JSONObject();
whereJson.put("material_id", meMaterialbase.getMaterial_id());
whereJson.put("material_code", meMaterialbase.getMaterial_code());
whereJson.put("stor_code", "BGK");
whereJson.put("sect_code", "BG01");
whereJson.put("stor_code", "YL");
whereJson.put("sect_code", "YL01");
whereJson.put("siteCode", "CKD01");
StrategyMater mater = new StrategyMater();
mater.setQty(BigDecimal.valueOf(outboundTaskRequest.getQty()));
@@ -214,19 +221,33 @@ public class VoiceCallbackService {
return null;
};
public JSONObject handleMoveTask(MoveTaskRequest moveTaskRequest){
String taskId = applyTaskMap.get("StructMoveTask").create((JSONObject) JSONObject.toJSON(moveTaskRequest));
Structattr attrDao = iStructattrService.findByCode(moveTaskRequest.getLocationFrom());
JSONObject taskForm = new JSONObject();
taskForm.put("point_code1", moveTaskRequest.getLocationFrom());
taskForm.put("point_code", moveTaskRequest.getLocationTo());
taskForm.put("config_code", IOSConstant.MOVE_CONFIG_TASK);
taskForm.put("vehicle_code", attrDao.getStoragevehicle_code());
String taskId = applyTaskMap.get("MoveTask").create((JSONObject) JSONObject.toJSON(taskForm));
return null;
};
public JSONObject handleQueryTask(QueryBoundRequest queryBoundRequest){
List<StructattrVechielDto> structCode = iStructattrService.collectVechicle(MapOf.of("struct_code", queryBoundRequest.getLocation(), "search", queryBoundRequest.getMaterial()));
Map<String, BigDecimal> collect = structCode.stream()
.collect(Collectors.groupingBy(StructattrVechielDto::getMaterial_code,
Collectors.reducing(BigDecimal.ZERO, StructattrVechielDto::getQty, BigDecimal::add)));
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("当前仓库物料有物料");
for (StructattrVechielDto vechielDtos : structCode) {
stringBuffer.append(vechielDtos.getMaterial_name()).append("共计")
.append(vechielDtos.getQty())
collect.entrySet().forEach(r -> {
stringBuffer.append(r.getKey()).append("共计")
.append(r.getValue())
.append("个");
}
});
// for (StructattrVechielDto vechielDtos : structCode) {
// stringBuffer.append(vechielDtos.getMaterial_name()).append("共计")
// .append(vechielDtos.getQty())
// .append("个");
// }
String text = stringBuffer.toString();
try {
// voiceService.speechTts(text);

27
nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/speech/api/AsrTtsClient.java
View File

@@ -18,6 +18,7 @@ public class AsrTtsClient {
@Value("${vc.base-url:http://127.0.0.1:5000}")
private String vcBaseUrl;
/**
* 健康检查
*/
@@ -29,12 +30,14 @@ public class AsrTtsClient {
});
}
/**
* 语音转文字
*/
public static void asr() {
}
/**
* 文字转语音
*/
@@ -55,20 +58,19 @@ public class AsrTtsClient {
});
}
/**
* 用户确认文字
*/
public CompletableFuture<Object> confirm(String text, boolean confirmed) {
return CompletableFuture.supplyAsync(() -> {
Map<String, Object> params = new HashMap<>();
params.put("text", text);
params.put("confirmed", confirmed);
return HttpRequest.post(vcBaseUrl + "/api/speech/confirm")
.body(JSONObject.toJSONString(params))
.contentType("application/json")
.execute()
.body();
});
public String confirm(String text, boolean confirmed) {
Map<String, Object> params = new HashMap<>();
params.put("text", text);
params.put("confirmed", confirmed);
return HttpRequest.post(vcBaseUrl + "/api/speech/confirm")
.body(JSONObject.toJSONString(params))
.contentType("application/json")
.execute()
.body();
}
@@ -76,7 +78,8 @@ public class AsrTtsClient {
return CompletableFuture.supplyAsync(() -> {
return HttpRequest.post(vcBaseUrl + "/api/speech/asr")
.form("audio", pcmData)
.form("audio", pcmData, "audio.pcm")
// .body(pcmData)
.contentType("multipart/form-data")
.execute()
.body();

21
nladmin-system/nlsso-server/src/main/java/org/nl/wms/system_manage/service/speech/impl/SysSpeechServiceImpl.java
View File

@@ -1,5 +1,6 @@
package org.nl.wms.system_manage.service.speech.impl;
import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
import org.nl.common.utils.MapOf;
import org.nl.wms.system_manage.service.speech.SysSpeechService;
@@ -39,18 +40,20 @@ public class SysSpeechServiceImpl implements SysSpeechService {
@Override
public Map<String, Object> confirm(String text) {
if ("".equals( text)){
if ("".equals(text)) {
throw new RuntimeException("确认文本为空");
}
asrTtsClient.confirm(text, true)
.thenAccept(result -> {
log.info("语音控制：用户确认结果: {}", result);
}).exceptionally(ex -> {
log.error("语音控制：用户确认异常: {}", ex);
return null;
});
String resJson = asrTtsClient.confirm(text, true);
log.info("语音控制：用户确认结果: {}", resJson);
//TODO
JSONObject jsonObject = JSONObject.parseObject(resJson);
JSONObject data = jsonObject.getJSONObject("data");
if (data != null) {
String resText = data.getString("text");
return MapOf.of("text", resText);
}
byte[] audio = asrTtsClient.tts(text).join();
//TODO 音频返回前端播放处理
return Collections.emptyMap();
return MapOf.of("text", text);
}
}

9
nladmin-system/nlsso-server/src/main/resources/config/application-dev.yml
View File

@@ -9,10 +9,11 @@ spring:
druid:
db-type: com.alibaba.druid.pool.DruidDataSource
driverClassName: com.mysql.cj.jdbc.Driver
url: jdbc:mysql://${DB_HOST:192.168.81.251}:${DB_PORT:3306}/${DB_NAME:wms_standardv2}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true&useSSL=false
# url: jdbc:mysql://${DB_HOST:localhost}:${DB_PORT:3306}/${DB_NAME:wms_oulun}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true&useSSL=false
# url: jdbc:mysql://${DB_HOST:192.168.81.251}:${DB_PORT:3306}/${DB_NAME:wms_standardv2}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true&useSSL=false
url: jdbc:mysql://${DB_HOST:localhost}:${DB_PORT:3306}/${DB_NAME:wms_standardv2}?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false&useOldAliasMetadataBehavior=true&allowPublicKeyRetrieval=true&useSSL=false
username: ${DB_USER:root}
password: ${DB_PWD:P@ssw0rd.}
password: ${DB_PWD:123456}
# password: ${DB_PWD:P@ssw0rd.}
# 初始连接数
initial-size: 15
# 最小连接数
@@ -137,3 +138,5 @@ sa-token:
lucene:
index:
path: D:\lms\lucene\index
vc:
base-url: http://192.168.20.156:5000

153
nladmin-ui/src/components/VoiceInputDialog/index.vue
View File

@@ -133,10 +133,18 @@
<span>正在录音... {{ recordingDuration }}"</span>
</div>
<!-- 识别中提示 -->
<div v-if="isTranscribing" class="transcribing-hint">
<div class="transcribing-spinner">
<i class="el-icon-loading" />
</div>
<span>正在识别录音内容...</span>
</div>
<!-- 录音按钮 -->
<div
class="record-button"
:class="{ 'recording': isRecording, 'disabled': !isConnected || isProcessing }"
:class="{ 'recording': isRecording, 'disabled': !isConnected || isProcessing || isTranscribing }"
@mousedown="startRecording"
@mouseup="stopRecording"
@mouseleave="isRecording && stopRecording()"
@@ -240,7 +248,7 @@ export default {
this.isConnected = false
},
// 处理WebSocket消息
// 处理WebSocket消息 - 只处理user和system两种类型
handleWebSocketMessage(data) {
// 判断数据类型
if (data instanceof ArrayBuffer || data instanceof Blob) {
@@ -252,14 +260,21 @@ export default {
const message = JSON.parse(data)
console.log('收到WebSocket消息:', message)
if (message.type === 'error') {
this.handleError(message.message || '识别失败')
} else if (message.type === 'transcription') {
// 语音识别结果
this.handleTranscriptionResult(message)
} else if (message.type === 'response') {
// 系统文本回复
this.handleSystemTextResponse(message)
// 只处理user和system两种类型
if (message.type === 'user') {
// 用户识别结果消息
this.handleUserMessage(message)
} else if (message.type === 'system') {
// 系统回复消息
this.handleSystemMessage(message)
} else if (message.type === 'error') {
// 错误消息作为系统消息显示
this.addMessage({
role: 'system',
type: 'text',
content: message.message || '处理失败',
timestamp: Date.now()
})
}
} catch (e) {
console.error('解析WebSocket消息失败:', e)
@@ -267,9 +282,9 @@ export default {
}
},
// 处理语音识别结果
handleTranscriptionResult(message) {
const text = message.text || message.transcript || ''
// 处理用户消息（语音识别结果）
handleUserMessage(message) {
const text = message.message
if (!text) return
this.isTranscribing = false
@@ -284,6 +299,19 @@ export default {
})
},
// 处理系统消息
handleSystemMessage(message) {
this.isProcessing = false
// 添加系统消息
this.addMessage({
role: 'system',
type: 'text',
content: message.message,
timestamp: Date.now()
})
},
// 确认消息 - 用户点击确认按钮后执行
async confirmMessage(message) {
if (!message || !message.content) return
@@ -294,14 +322,16 @@ export default {
// 执行confirm函数
try {
const res = await confirm(message.content)
const res = await confirm({
text: message.content
})
this.isProcessing = false
// 添加系统回复消息
this.addMessage({
role: 'system',
type: 'text',
content: res.data || '收到您的消息',
content: res.text,
timestamp: Date.now()
})
} catch (error) {
@@ -328,18 +358,7 @@ export default {
this.scrollToBottom()
},
// 处理系统文本回复
handleSystemTextResponse(message) {
this.isProcessing = false
this.addMessage({
role: 'system',
type: 'text',
content: message.text || message.content || '收到您的消息',
timestamp: Date.now()
})
},
// 处理系统语音回复
// 处理系统语音回复（二进制数据）
async handleSystemVoiceResponse(audioData) {
try {
// 创建音频URL
@@ -360,7 +379,7 @@ export default {
role: 'system',
type: 'voice',
audioUrl: audioUrl,
duration: '3"', // 这里应该根据实际音频时长计算
duration: '3"',
timestamp: Date.now()
}
this.addMessage(message)
@@ -372,27 +391,7 @@ export default {
}
},
// 处理错误
handleError(message) {
this.isProcessing = false
this.isTranscribing = false
this.addMessage({
role: 'system',
type: 'text',
content: message || '服务出现错误',
timestamp: Date.now()
})
// 更新最后一条用户消息状态
// const lastUserMessage = this.messages.filter(m => m.role === 'user').pop()
// if (lastUserMessage) {
// lastUserMessage.isTranscribing = false
// lastUserMessage.content = message
// lastUserMessage.type = 'text'
// }
//
// this.$message.error(message)
},
// 添加消息
addMessage(message) {
@@ -1006,31 +1005,51 @@ export default {
border-top: 1px solid #ebeef5;
background: #fff;
.recording-hint {
display: flex;
align-items: center;
justify-content: center;
margin-bottom: 10px;
color: #F56C6C;
font-size: 14px;
.recording-wave {
.recording-hint {
display: flex;
align-items: center;
margin-right: 8px;
justify-content: center;
margin-bottom: 10px;
color: #F56C6C;
font-size: 14px;
span {
width: 4px;
height: 20px;
background: #F56C6C;
margin: 0 2px;
border-radius: 2px;
animation: waveAnimation 0.5s ease-in-out infinite;
.recording-wave {
display: flex;
align-items: center;
margin-right: 8px;
span {
width: 4px;
height: 20px;
background: #F56C6C;
margin: 0 2px;
border-radius: 2px;
animation: waveAnimation 0.5s ease-in-out infinite;
}
}
}
}
.record-button {
.transcribing-hint {
display: flex;
align-items: center;
justify-content: center;
margin-bottom: 10px;
color: #409EFF;
font-size: 14px;
.transcribing-spinner {
display: flex;
align-items: center;
margin-right: 8px;
i {
font-size: 18px;
animation: rotating 1s linear infinite;
}
}
}
.record-button {
display: flex;
align-items: center;
justify-content: center;