基于视频理解的智能视频剪辑指南

    model = whisper.load_model(
        model_type, download_root="/opt/ml/code/models/", in_memory=True, device="cuda"
    )
    result = model.transcribe(source, **decode_options)

    # language detection
    audio = whisper.load_audio(source)
    mel = whisper.log_mel_spectrogram(audio, model.dims.n_mels, padding=whisper.audio.N_SAMPLES).to(model.device)
    mel_segment = whisper.pad_or_trim(mel, whisper.audio.N_FRAMES)
    _, probs = model.detect_language(mel_segment)
    lang = max(probs, key=probs.get)

from funasr import AutoModel as FunasrModel
from silero_vad import get_speech_timestamps, load_silero_vad, read_audio

if lang == "zh":
    vad_model = FunasrModel(
        model="/opt/ml/code/models/vad_model", model_revision="v2.0.4"
    )
    res = vad_model.generate(input=source)
    speech_timestamps = []
    for row in res[0]["value"]:
        speech_timestamps.append({"start": row[0] / 1000, "end": row[1] / 1000})
else:
    vad_model = load_silero_vad()
    wav = read_audio(source)
    speech_timestamps = get_speech_timestamps(
        wav,
        vad_model,
        return_seconds=True,
        threshold=0.2,  # Return speech timestamps in seconds (default is samples)
    )

def get_sample_points(start, end, sample_count=12):
    step = (end - start) / (sample_count + 1)
    center = (start + end) / 2
    sample_points = []

    for i in range(1, sample_count + 1):
        point = start + i * step
        if point == center:
            rounded_point = int(round(point))
        elif point < center:
            rounded_point = int(math.ceil(point))
        else:
            rounded_point = int(math.floor(point))
        sample_points.append(rounded_point)

    return list(set(sample_points))
parameter_list = []
index = 0
for shot in shots:
    start_time = shot["StartTimestampMillis"] / 1000
    duration = (shot["EndTimestampMillis"] - shot["StartTimestampMillis"]) / 1000

    video_bytes = process_segment(temp_file_path, start_time, duration)

    # 上传LLM payload到S3，而不是原始视频
    video_key = f"transcribe_video/{project_id}/{shot['StartFrameNumber']}.json"
    s3_client.put_object(
        Bucket=bucket_name,
        Key=video_key,
        Body=json.dumps(build_llm_payload(get_content_messages(video_bytes, "video"), language)),
    )

    parameter_list.append(
        {
            "index": index,
            "start_time": str(round(shot["StartTimestampMillis"] / 1000, 2)),
            "end_time": str(round(shot["EndTimestampMillis"] / 1000, 2)),
            "next_start_time": str(round(shot["next_start_time"] / 1000, 2)),
            "previous_end_time": str(round(shot["previous_end_time"] / 1000, 2)),
            "parameter_key": video_key,
        }
    )
    index += 1

You are a professional video editor, skilled in analyzing video shot sequences and extracting their core content. I will provide you with a series of video shot descriptions in chronological order. Your task is to create a summary describing what the video is about. 
Note that this summary needs to be completed in the same language as the shot description. If the shot description is in English, then the summary you create must also be in English. 
Please carefully read the following shot descriptions:
<document>
${shot_str}
</document>
Please follow the steps below:

1. Read through the sequence of shot descriptions carefully to ensure understanding of the overall content and the connections between shots.
2. Extract the core scenes and key information from the lens sequence.
3. Organize these core scenarios and key information into a coherent summary. The summary content needs to be structured.
4. Please use the ${language} language for output.
Please only output the summary content, do not add any additional explanations or comments.

"You are a professional text editor, skilled in segmenting long texts and summarizing topics.
 Please carefully read the following sentences and segment the content according to the given rules.
  I will edit the video based on your segmentation results.\n\nNote that these segments and summaries need to be completed in the same language as the text. 
  If the text is in English, then the segments and summaries you create must also be in English
 .\n\nsentence:\n<document>\n${data}\n</document>\n\n\nPlease follow the following guidelines:\n\n1. Combine consecutive, thematically related sentences into one concise summary of the topic.\n2. 
 Ensure the topic is concise, clear, and engaging.\n3. Output the summarized topics and corresponding sentence number ranges in order, using a hyphen to connect the starting and ending sentence numbers (e.g., 1-10).\n4. Sentences within each paragraph must be continuous, and cross-sentence segmentation is not allowed.\n5. All sentences must be included in paragraphs.\n6. Please output in the original language.\n7. Do not use any single or double quotes in the summary.\n\nPlease summarize into a minimum of 1 and a maximum of 4 paragraphs\n\nPlease strictly follow the following JSON format to output your segmented results:\n\n[\n{\"start\": <starting sentence number>, \"end\": <ending sentence number>, \"seg\": <paragraph number>, \"summary\": \"<summarized paragraph content>\"}\n{\"start\": <start sentence number>, \"end\": <end sentence number>, \"seg\": <paragraph number>, \"summary\": \"<summarized paragraph content>\"}\n...\n]\nNote:\n\n- The paragraph number (seg) must be a consecutive integer.\nPlease ensure that all content from sentence 1 to sentence 658 is included in your segment.\n- Only output the content in JSON format, no other explanations are needed.\n\nPlease remember that high-quality segmentation and summarization are crucial for the subsequent video editing work. We look forward to your professional performance!

def lambda_handler(event, context):
    if event["segment"]["payload"].get("error"):
        raise Exception(event["segment"]["payload"].get("error"))
    transcribe_job_id = event["transcribe_job_id"]
    job_id = event["job_id"]
    segment = event["segment"]["payload"].get("body")
    if segment.startswith("```json") and segment.endswith("```"):
        segment = segment[7:-3]

    try:
        segment_dict = json.loads(segment)
    except json.decoder.JSONDecodeError:
        logger.info(segment)
        segment_dict = parse_json(segment)
    sentences = get_all_sentences(transcribe_job_id)
    i = 0
    with segment_table.batch_writer() as batch:
        for item in segment_dict:
            segment = {}
            segment["start_time"] = sentences[item["start"] - 1]["start_time"]
            segment["end_time"] = sentences[item["end"] - 1]["end_time"]
            segment["summary"] = item["summary"]
            segment["job_id"] = job_id
            segment["index"] = i
            i = i + 1
            batch.put_item(Item=segment)

# Determine which prompt to use based on the current step
if current_highlight_step == 0:
    # Initial selection - ask for complete highlight selection
    task_prompt = INITIAL_TASK_PROMPT
    format_prompt = INITIAL_FORMAT_PROMPT.format(second_limit=second_limit)
elif highlight_second > current_highlight_duration:
    # Need to add more shots
    diff_time = round(highlight_second - current_highlight_duration, 2)
    task_prompt = INCREMENTAL_TASK_PROMPT
    format_prompt = INCREMENTAL_FORMAT_PROMPT.format(
        current_highlight_duration=current_highlight_duration,
        diff_time=diff_time,
        current_highlight_shots=current_highlight_shots,
    )
else:
    # Need to remove some shots
    diff_time = round(current_highlight_duration - highlight_second, 2)
    task_prompt = DECREMENTAL_TASK_PROMPT
    format_prompt = DECREMENTAL_FORMAT_PROMPT.format(
        current_highlight_duration=current_highlight_duration,
        diff_time=diff_time,
        current_highlight_shots=current_highlight_shots,
    )

你是一名专业的视频剪辑师。这是你需要剪辑的视频介绍：
<video_intro>
${summary_str}
</video_intro>

以下是一系列按时间顺序排列的视频镜头，包括用<shot_description>标记的对话内容和<duration>标记的镜头时长（秒）。
<shot_sequence>
${shot_str}
</shot_sequence>

你的任务是${task_prompt}

剪辑出高光时刻，高光时刻只需要包括新产品科技发布， 精彩的示例
严格按照原片的时间顺序排列这些不调整先后顺序
请注意，一个高光时刻可能跨越多个连续镜头。为保持对话的连贯性，你可以将这些相关镜头都包含在高光视频中。
为了连续性，在选择高光时刻时请尽可能包括多个连续的镜头。对话内容不要太突兀。尽量保持连贯性。如果一个镜头小于5秒，且前后的镜头都没有被选为高光时刻，那么它也不该被选中。
每个高光时刻包含的若干个连续镜头中的对话应该是完整的，不应该突然跳转到另一个高光时刻。
请剔除片头曲和片尾曲 ${format_prompt}
请删除头尾空白部分

你是一位专业的文本编辑，擅长分割长文本并概括主题。请仔细阅读以下句子，并根据给定规则进行分段。
我将根据你的分段结果编辑视频。
注意：这些分段和概括需要用与文本相同的语言完成。
如果文本是英语，则你创建的分段和概括也必须使用英语。
请遵循以下指南：
1. 将连续的主题相关句子合并为一个简洁的主题概述。
2. 确保主题简洁、清晰且引人入胜。
3. 输出概括的主题和对应的句子编号范围，使用连字符连接起始和结束句子号（例如 1-10）。
4. 每个段落内的句子必须连续，跨句子分段不允许。
5. 所有句子必须包含在段落中。
6. 请以原始语言输出。
7. 不要在概括中使用任何单引号或双引号。
请概括为最小1个和最大4个段落。请严格遵循以下JSON
{\"start\": <starting sentence number>, \"end\": <ending sentence number>, \"seg\": <paragraph number>, \"summary\": \"<summarized paragraph content>\"}\n{\"start\": <start sentence number>, \"end\": <end sentence number>, \"seg\": <paragraph number>, \"summary\": \"<summarized paragraph content>\"}\n...\n]\n
注意：
- 段落号（seg）必须是连续整数。请确保从句子1到句子658的所有内容都包含在你的分段中。
- 仅输出JSON格式内容，不需要其他说明。