Qwen3.8-Omni-Flash：耳聪目明，办事得力

"""
Environment variables:
  DASHSCOPE_API_KEY: Your API Key from https://platform.qianwenai.com/home
  DASHSCOPE_BASE_URL: (optional) Base URL for compatible-mode API.
    - Beijing: https://dashscope.aliyuncs.com/compatible-mode/v1
    - Singapore: https://dashscope-intl.aliyuncs.com/compatible-mode/v1
"""
from openai import OpenAI
import os

api_key = os.environ.get("DASHSCOPE_API_KEY")
if not api_key:
    raise ValueError(
        "DASHSCOPE_API_KEY is required. "
        "Set it via: export DASHSCOPE_API_KEY='your-api-key'"
    )

client = OpenAI(
    api_key=api_key,
    base_url=os.environ.get(
        "DASHSCOPE_BASE_URL",
        "https://dashscope.aliyuncs.com/compatible-mode/v1",
    ),
)

messages=[
    {
        "role": "user",
        "content": [
            {
                "type": "image_url",
                "image_url": {
                    "url": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20241022/emyrja/dog_and_girl.jpeg"
                },
            },
            {
                "type": "input_audio",
                "input_audio": {
                    "data": "https://help-static-aliyun-doc.aliyuncs.com/file-manage-files/zh-CN/20250211/tixcef/cherry.wav",
                    "format": "wav"
                },
            },
            {"type": "text", "text": "请描述图片内容，并告诉我音频在说什么。"},
        ],
    },
]

completion = client.chat.completions.create(
    model="qwen3.8-omni-flash",
    messages=messages,
    extra_body={
        "enable_thinking": True,
        # "preserve_thinking": True,
    },
    reasoning_effort="xhigh",  # supported levels are xhigh, medium, and low
    stream=True,
)

reasoning_content = ""
answer_content = ""
is_answering = False
print("\n" + "=" * 20 + "Reasoning" + "=" * 20 + "\n")

for chunk in completion:
    if not chunk.choices:
        print("\nUsage:")
        print(chunk.usage)
        continue

    delta = chunk.choices[0].delta

    if hasattr(delta, "reasoning_content") and delta.reasoning_content is not None:
        if not is_answering:
            print(delta.reasoning_content, end="", flush=True)
        reasoning_content += delta.reasoning_content

    if hasattr(delta, "content") and delta.content:
        if not is_answering:
            print("\n" + "=" * 20 + "Answer" + "=" * 20 + "\n")
            is_answering = True
        print(delta.content, end="", flush=True)
        answer_content += delta.content

Provide a detailed description of the video.

Make sure your description covers every one of the following dimensions:

Visual
- Subjects and characters: appearance, clothing, gender/age cues, identity, distinctive features
- Actions and events in chronological order, and how the scene evolves over time
- Setting and background: location, environment, time of day
- Spatial layout and relations between subjects/objects; counts and quantities
- On-screen text: captions, titles, subtitles, logos, UI — exact content and appearance
- Visual style: colors, lighting, camera shots, angles, and camera movement

Audio
- Speech: the exact spoken content, transcribed verbatim
- Speakers: who is speaking (mapped to the on-screen person or voice-over), with accent, tone, gender/age cues
- Speaking state: prosody, emotion, volume, and speaking style
- Music: presence, genre/mood, and lyrics if any
- Sound effects and ambient/background sounds
- Non-speech vocalizations: laughter, crying, applause, etc.

Audio-visual correspondence
- Which speech or sound aligns with which on-screen person or visual event
- The timing of each event, expressed with timestamps

It should explicitly include three sections:

1. A structured chronological storyline of **every noticeable audio and visual details**
2. A structured list of all visible text. For each text element, include start timestamp, end timestamp, the exact text content, the appearance characteristics. If no text appears, explicitly state so.
3. A structured speech-to-text transcription, include speaker (corresponding to the character or voice-over in Section 1, including their accent and tone), exact spoken content, start timestamp, end timestamp, and speaking state (prosody, emotion, and style). If no speech appears, explicitly state so.

Aside from these three required sections, you are free to organize any additional content in any way you find helpful. This additional content can include global information about the entire video or localized information about specific moments. You may choose the topic of this extra content freely.

Rules:

- Add as much descriptive detail as possible.
- Do not use Markdown bold formatting.
- Carefully look at frames and listen to the audio, making sure no detail is overlooked.

Output Format:
```
## Storyline

<xx:xx.xxx> - <xx:xx.xxx>
<an unstructured long paragraph in natural language describing what happened during this period, blending both audio and video details.>

<xx:xx.xxx> - <xx:xx.xxx>
<an unstructured long paragraph in natural language describing what happened during this period, blending both audio and video details.>

<xx:xx.xxx> - <xx:xx.xxx>
<an unstructured long paragraph in natural language describing what happened during this period, blending both audio and video details.>

...

## Visible Text

<xx:xx.xxx> - <xx:xx.xxx>
“<element>”: <appearance>
“<element>”: <appearance>

<xx:xx.xxx> - <xx:xx.xxx>
“<element>”: <appearance>
“<element>”: <appearance>
“<element>”: <appearance>

<xx:xx.xxx> - <xx:xx.xxx>
“<element>”: <appearance>

...

## Speakers and Transcript

Speaker profiles:
<speaker> - <profile>
<speaker> - <profile>
<speaker> - <profile>
...

<xx:xx.xxx> - <xx:xx.xxx>
Speaker: <speaker>
State: <description>
Content: “<content>”

<xx:xx.xxx> - <xx:xx.xxx>
Speaker: <speaker>
State: <description>
Content: “<content>”

<xx:xx.xxx> - <xx:xx.xxx>
Speaker: <speaker>
State: <description>
Content: “<content>”

...

## <another section>

<paragraphs>

## <another section>

<paragraphs>

...
```

Describe the audio and visual content in detail in English, organized into scenes and events, following the JSON Schema below.
All timestamps must be relative to the beginning of the video. End times must not precede start times or exceed the video duration. Each event must fall within the time range of its parent scene.
Include only information directly supported by the audio or video. Do not guess or invent details. Do not infer causality merely because a sound and an action occur at the same time.
Return only valid JSON, without Markdown fences or commentary.

JSON Schema:
{
  "$defs": {
    "Event": {
      "additionalProperties": false,
      "properties": {
        "time_range": {
          "$ref": "#/$defs/TimeRange",
          "description": "Time range of the event"
        },
        "participants": {
          "description": "People, animals, or objects involved, named by observable features; use consistent names for the same participant",
          "items": {
            "type": "string"
          },
          "title": "Participants",
          "type": "array"
        },
        "action": {
          "description": "Specific actions, interactions, and observable outcomes",
          "title": "Action",
          "type": "string"
        },
        "sounds": {
          "description": "Sounds heard during the event; use an empty list if none are discernible",
          "items": {
            "type": "string"
          },
          "title": "Sounds",
          "type": "array"
        }
      },
      "required": [
        "time_range",
        "participants",
        "action",
        "sounds"
      ],
      "title": "Event",
      "type": "object"
    },
    "Scene": {
      "additionalProperties": false,
      "properties": {
        "time_range": {
          "$ref": "#/$defs/TimeRange",
          "description": "Time range of the scene"
        },
        "setting": {
          "description": "Environment, spatial layout, and main visual features",
          "title": "Setting",
          "type": "string"
        },
        "events": {
          "description": "Events in chronological order; use an empty list if there are none",
          "items": {
            "$ref": "#/$defs/Event"
          },
          "title": "Events",
          "type": "array"
        }
      },
      "required": [
        "time_range",
        "setting",
        "events"
      ],
      "title": "Scene",
      "type": "object"
    },
    "TimeRange": {
      "additionalProperties": false,
      "properties": {
        "start_seconds": {
          "description": "Start time in seconds relative to the beginning of the video",
          "minimum": 0,
          "title": "Start Seconds",
          "type": "number"
        },
        "end_seconds": {
          "description": "End time in seconds; must not precede the start time",
          "minimum": 0,
          "title": "End Seconds",
          "type": "number"
        }
      },
      "required": [
        "start_seconds",
        "end_seconds"
      ],
      "title": "TimeRange",
      "type": "object"
    }
  },
  "additionalProperties": false,
  "properties": {
    "summary": {
      "description": "An overview of the main content of the video",
      "title": "Summary",
      "type": "string"
    },
    "scenes": {
      "description": "Scenes in chronological order; group continuous footage with a consistent setting into one scene",
      "items": {
        "$ref": "#/$defs/Scene"
      },
      "title": "Scenes",
      "type": "array"
    }
  },
  "required": [
    "summary",
    "scenes"
  ],
  "title": "CaptionResult",
  "type": "object"
}

帮我安装一下 https://github.com/QwenLM/Qwen-MM-Plugins 的 core、api 以及 omni 相关的插件。

# 运行官方引导安装器：
curl -fsSL https://raw.githubusercontent.com/QwenLM/Qwen-MM-Plugins/main/install.sh | bash

@song.mp3 根据歌曲节奏和内容生成一个完整的 MV。
@short_drama.mp4 将视频翻译成英文，并尽量保留原说话人的声音特征。
@movie.mp4 制作一个带中文旁白和字幕的影视解说视频。

@weekly_report_sop.mp4 帮我把这个写周报的录屏整理成 Skill.md。
@tutorial.mp4 将教程整理成带关键截图、时间戳和步骤说明的 PDF 笔记。
@documentary.mp4 建立音画记忆，记录人物、对话、声音和重要事件。

# Run pip install websocket-client pyaudio dashscope opencv-python -U to install dependencies
"""
Environment variables:
  DASHSCOPE_API_KEY: Your API Key from https://platform.qianwenai.com/home
  DASHSCOPE_BASE_URL: (optional) Base URL for realtime API.
    - Beijing: wss://dashscope.aliyuncs.com/api-ws/v1/realtime
    - Singapore: wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime
"""
import os
import base64
import time
import pyaudio
import cv2
from dashscope.audio.qwen_omni import MultiModality, AudioFormat,OmniRealtimeCallback,OmniRealtimeConversation
import dashscope

url = f'wss://dashscope.aliyuncs.com/api-ws/v1/realtime'
dashscope.api_key = os.getenv('DASHSCOPE_API_KEY')
# Determine the voice
voice = 'Tina'
# Determine the model
model = 'qwen3.8-omni-flash-realtime'
# Determine the model role
instructions = "You are Qwen-Omni, a helpful assistant."
video_fps = 1
video_size = (1280, 720)
class SimpleCallback(OmniRealtimeCallback):
    def __init__(self, pya):
        self.pya = pya
        self.out = None
    def on_open(self):
        # Initialize audio output stream
        self.out = self.pya.open(
            format=pyaudio.paInt16,
            channels=1,
            rate=24000,
            output=True
        )
    def on_event(self, response):
        if response['type'] == 'response.audio.delta':
            # Play audio
            self.out.write(base64.b64decode(response['delta']))
        elif response['type'] == 'conversation.item.input_audio_transcription.delta':
            # Streaming preview: text is the confirmed prefix, stash is the confirmed suffix
            preview = response.get('text', '') + response.get('stash', '')
            print(f"\r[User] {preview}", end='', flush=True)
        elif response['type'] == 'conversation.item.input_audio_transcription.completed':
            # Transcription completed, print the final text and a new line
            print(f"\r[User] {response['transcript']}")
        elif response['type'] == 'response.audio_transcript.done':
            # Print the assistant's response text
            print(f"[LLM] {response['transcript']}")

# 1. Initialize audio device
pya = pyaudio.PyAudio()
# 2. Create callback function and session
callback = SimpleCallback(pya)
conv = OmniRealtimeConversation(model=model, callback=callback, url=url)
# 3. Establish connection and configure session
conv.connect()
conv.update_session(output_modalities=[MultiModality.AUDIO, MultiModality.TEXT], voice=voice, instructions=instructions)
# 4. Initialize audio input stream
mic = pya.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True)
camera = cv2.VideoCapture(0)
next_frame_at = 0.0
# 5. Main loop to process audio and video input
try:
    if not camera.isOpened():
        raise RuntimeError("Cannot open camera 0.")
    camera.set(cv2.CAP_PROP_FRAME_WIDTH, video_size[0])
    camera.set(cv2.CAP_PROP_FRAME_HEIGHT, video_size[1])
    camera.set(cv2.CAP_PROP_BUFFERSIZE, 1)
    print(f"Conversation started with camera ({video_fps} fps, {video_size[0]}x{video_size[1]}), speak into the microphone (Ctrl+C to exit)...")
    while True:
        audio_data = mic.read(3200, exception_on_overflow=False)
        conv.append_audio(base64.b64encode(audio_data).decode())
        success, frame = camera.read()
        if not success:
            raise RuntimeError("Cannot read a camera frame.")
        if time.monotonic() >= next_frame_at:
            frame = cv2.resize(frame, video_size)
            success, image = cv2.imencode('.jpg', frame, [cv2.IMWRITE_JPEG_QUALITY, 80])
            if not success:
                raise RuntimeError("Cannot encode a camera frame.")
            conv.append_video(base64.b64encode(image).decode())
            next_frame_at = time.monotonic() + 1 / video_fps
        time.sleep(0.01)
except KeyboardInterrupt:
    pass
finally:
    # Clean up resources
    camera.release()
    conv.close()
    mic.close()
    if callback.out:
        callback.out.close()
    pya.terminate()
    print("\nConversation ended")

npm install -g qwen-live-harness
qwen-live-harness init
qwen-live-harness

@misc{qwen38omniflash,
    title = {Qwen3.8-Omni-Flash: Omni Senses. Agentic Delivery.},
    url = {https://qwen.ai/blog?id=qwen3.8-omni-flash},
    author = {{Qwen Team}},
    month = {September},
    year = {2026}
}