Recherche avancée

Sur d’autres sites (16449)

• extracting video and data streams from MPEG2 TS over RTP in real-time

  10 janvier 2024, par Tejal Barnwal

  I have H264 video stream and KLV meta data encapsulated inside MPEG2 TS container which are sent over an RTP over UDP from a camera.
I intend to do the following :

  


  

  1. Extract both video and data streams from RTP

  


  2. Process video feed using opencv in a seperate thread

  


  3. process klv metadata in a seperate thread

  


  


  My problem what exact arguments should I provide to ffmpeg so as to read h264 video stream and show the images frame by frame using opencv ?

  


  With the help of some previous posts like Simultaneously map video and data streams to one subprocess pipeline in real-time, I was able to get some idea about how could I proceed to procees the stream over RTP.

  


  I started out by using the following script :

  


  #!/usr/bin/env python3
from asyncio import streams
from logging.handlers import QueueListener
import klvdata
import subprocess as sp
import shlex
import threading
import numpy as np
import cv2
import time
from io import BytesIO

# Video reader thread.
def video_reader(pipe):
    cols, rows = 1280, 720  # Assume we know frame size is 1280x720

    counter = 0
    while True:
        print("read image")
        raw_image = pipe.read(cols*rows*3)  # Read raw video frame

        # Break the loop when length is too small
        if len(raw_image) < cols*rows*3:
            break

        if (counter % 10) == 0:
            # Show video frame evey 60 frames
            image = np.frombuffer(raw_image, np.uint8).reshape([rows, cols, 3])
            cv2.imshow('Video', image) # Show video image for testing
            cv2.waitKey(1)
        counter += 1
        print("image showed on window")
        time.sleep(0.25)



# https://github.com/paretech/klvdata/tree/master/klvdata
def bytes_to_int(value, signed=False):
    """Return integer given bytes."""
    return int.from_bytes(bytes(value), byteorder='big', signed=signed)


# Data reader thread (read KLV data).
def data_reader(pipe):
    key_length = 16  # Assume key length is 16 bytes.

    f = open('data.bin', 'wb')  # For testing - store the KLV data to data.bin (binary file)

    while True:
        # https://en.wikipedia.org/wiki/KLV
        # The first few bytes are the Key, much like a key in a standard hash table data structure.
        # Keys can be 1, 2, 4, or 16 bytes in length.
        # Presumably in a separate specification document you would agree on a key length for a given application.
        key = pipe.read(key_length)  # Read the key
        
        if len(key) < key_length:
            break  # Break the loop when length is too small
        f.write(key)  # Write data to binary file for testing

        # https://github.com/paretech/klvdata/tree/master/klvdata
        # Length field
        len_byte = pipe.read(1)

        if len(len_byte) < 1:
            break  # Break the loop when length is too small
        f.write(len_byte)  # Write data to binary file for testing

        byte_length = bytes_to_int(len_byte)

        # https://github.com/paretech/klvdata/tree/master/klvdata                                                
        if byte_length < 128:
            # BER Short Form
            length = byte_length
            ber_len_bytes = b''
        else:
            # BER Long Form
            ber_len = byte_length - 128
            ber_len_bytes = pipe.read(ber_len)

            if len(ber_len_bytes) < ber_len:
                break  # Break the loop when length is too small
            f.write(ber_len_bytes)  # Write ber_len_bytes to binary file for testing

            length = bytes_to_int(ber_len_bytes)

        # Read the value (length bytes)
        value = pipe.read(length)
        if len(value) < length:
            break  # Break the loop when length is too small
        f.write(value)  # Write data to binary file for testing

        klv_data = key + len_byte + ber_len_bytes + value  # Concatenate key length and data
        klv_data_as_bytes_io = BytesIO(klv_data)  # Wrap klv_data with BytesIO (before parsing)

        # Parse the KLV data
        for packet in klvdata.StreamParser(klv_data_as_bytes_io): 
            metadata = packet.MetadataList()
            for key, value in metadata.items():
                print(key, value)
                
            print("\n") # New line

# Execute FFmpeg as sub-process
# Map the video to stderr and map the data to stdout
process = sp.Popen(shlex.split('ffmpeg -hide_banner -loglevel quiet '                        # Set loglevel to quiet for disabling the prints ot stderr
                               '-i "rtp://192.168.0.141:11024" '                                        # Input video "Day Flight.mpg"
                               '-map 0:v -c:v rawvideo -pix_fmt bgr24 -f:v rawvideo pipe:2 ' # rawvideo format is mapped to stderr pipe (raw video codec with bgr24 pixel format)
                               '-map 0:d -c copy -copy_unknown -f:d data pipe:1 '            # Copy the data without ddecoding.
                               '-report'),                                                   # Create a log file (because we can't the statuses that are usually printed to stderr).
                                stdout=sp.PIPE, stderr=sp.PIPE)


# Start video reader thread (pass stderr pipe as argument).
video_thread = threading.Thread(target=video_reader, args=(process.stderr,))
video_thread.start()

# Start data reader thread (pass stdout pipe as argument).
data_thread = threading.Thread(target=data_reader, args=(process.stdout,))
data_thread.start()


# Wait for threads (and process) to finish.
video_thread.join()
data_thread.join()
process.wait()



  


  With the above script, I was facing two issues :

  


  

  1. The second thread resulted in an attribute error

  


  


  Exception in thread Thread-2:
Traceback (most recent call last):
  File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.8/threading.py", line 870, in run
    self._target(*self._args, **self._kwargs)
  File "video_data_extraction.py", line 97, in data_reader
    print(packet.MetadataList())
AttributeError: 'UnknownElement' object has no attribute 'MetadataList'



  


  

  2. With this though I continuously able to see following output on the terminal regarding reading the images

  


  


  read image
image showed on window
read image
image showed on window
read image
image showed on window
read image
image showed on window
read image
image showed on window
read image
image showed on window


  


  The imshow windows wasnt updating properly ! It seemed stuck after a few frames.

  


  Further diving into the lane with the help of following command, I concluded that the video stream that I am reading has H264 encoding

  


  ffprobe -i rtp://192.168.0.141:11024 -show_streams -show_formats


  


  Output of the above command :

  


  ffprobe version 4.2.7-0ubuntu0.1 Copyright (c) 2007-2022 the FFmpeg developers
  built with gcc 9 (Ubuntu 9.4.0-1ubuntu1~20.04.1)
  configuration: --prefix=/usr --extra-version=0ubuntu0.1 --toolchain=hardened --libdir=/usr/lib/aarch64-linux-gnu --incdir=/usr/include/aarch64-linux-gnu --arch=arm64 --enable-gpl --disable-stripping --enable-avresample --disable-filter=resample --enable-avisynth --enable-gnutls --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libjack --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librsvg --enable-librubberband --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-lv2 --enable-omx --enable-openal --enable-opencl --enable-opengl --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libx264 --enable-shared
  libavutil      56. 31.100 / 56. 31.100
  libavcodec     58. 54.100 / 58. 54.100
  libavformat    58. 29.100 / 58. 29.100
  libavdevice    58.  8.100 / 58.  8.100
  libavfilter     7. 57.100 /  7. 57.100
  libavresample   4.  0.  0 /  4.  0.  0
  libswscale      5.  5.100 /  5.  5.100
  libswresample   3.  5.100 /  3.  5.100
  libpostproc    55.  5.100 / 55.  5.100
[rtp @ 0xaaaac81ecce0] PES packet size mismatch
    Last message repeated 62 times
[NULL @ 0xaaaac81f09b0] non-existing PPS 0 referenced
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[rtp @ 0xaaaac81ecce0] PES packet size mismatch
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[rtp @ 0xaaaac81ecce0] PES packet size mismatch
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] non-existing PPS 0 referenced
    Last message repeated 1 times
[h264 @ 0xaaaac81f09b0] decode_slice_header error
[h264 @ 0xaaaac81f09b0] no frame!
[rtp @ 0xaaaac81ecce0] PES packet size mismatch
    Last message repeated 187 times
Input #0, rtp, from 'rtp://192.168.0.141:11024':
  Duration: N/A, start: 1317.040656, bitrate: N/A
  Program 1 
    Stream #0:1: Video: h264 (Constrained Baseline) ([27][0][0][0] / 0x001B), yuv420p(progressive), 1280x720, 25 fps, 25 tbr, 90k tbn
    Stream #0:0: Data: klv (KLVA / 0x41564C4B)
Unsupported codec with id 100356 for input stream 0
[STREAM]
index=0
codec_name=klv
codec_long_name=SMPTE 336M Key-Length-Value (KLV) metadata
profile=unknown
codec_type=data
codec_tag_string=KLVA
codec_tag=0x41564c4b
id=N/A
r_frame_rate=0/0
avg_frame_rate=0/0
time_base=1/90000
start_pts=118533659
start_time=1317.040656
duration_ts=N/A
duration=N/A
bit_rate=N/A
max_bit_rate=N/A
bits_per_raw_sample=N/A
nb_frames=N/A
nb_read_frames=N/A
nb_read_packets=N/A
DISPOSITION:default=0
DISPOSITION:dub=0
DISPOSITION:original=0
DISPOSITION:comment=0
DISPOSITION:lyrics=0
DISPOSITION:karaoke=0
DISPOSITION:forced=0
DISPOSITION:hearing_impaired=0
DISPOSITION:visual_impaired=0
DISPOSITION:clean_effects=0
DISPOSITION:attached_pic=0
DISPOSITION:timed_thumbnails=0
[/STREAM]
[STREAM]
index=1
codec_name=h264
codec_long_name=H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10
profile=Constrained Baseline
codec_type=video
codec_time_base=1/50
codec_tag_string=[27][0][0][0]
codec_tag=0x001b
width=1280
height=720
coded_width=1280
coded_height=720
has_b_frames=0
sample_aspect_ratio=N/A
display_aspect_ratio=N/A
pix_fmt=yuv420p
level=31
color_range=unknown
color_space=unknown
color_transfer=unknown
color_primaries=unknown
chroma_location=left
field_order=progressive
timecode=N/A
refs=1
is_avc=false
nal_length_size=0
id=N/A
r_frame_rate=25/1
avg_frame_rate=25/1
time_base=1/90000
start_pts=118533659
start_time=1317.040656
duration_ts=N/A
duration=N/A
bit_rate=N/A
max_bit_rate=N/A
bits_per_raw_sample=8
nb_frames=N/A
nb_read_frames=N/A
nb_read_packets=N/A
DISPOSITION:default=0
DISPOSITION:dub=0
DISPOSITION:original=0
DISPOSITION:comment=0
DISPOSITION:lyrics=0
DISPOSITION:karaoke=0
DISPOSITION:forced=0
DISPOSITION:hearing_impaired=0
DISPOSITION:visual_impaired=0
DISPOSITION:clean_effects=0
DISPOSITION:attached_pic=0
DISPOSITION:timed_thumbnails=0
[/STREAM]
[FORMAT]
filename=rtp://192.168.0.141:11024
nb_streams=2
nb_programs=1
format_name=rtp
format_long_name=RTP input
start_time=1317.040656
duration=N/A
size=N/A
bit_rate=N/A
probe_score=100
[/FORMAT]


  


  Further, in the log output, I see a lot of statements in regard to missed packets and PES packet mismatch

  


  [rtp @ 0xaaaaf31896c0] max delay reached. need to consume packet
[rtp @ 0xaaaaf31896c0] RTP: missed 98 packets
[rtp @ 0xaaaaf31896c0] Continuity check failed for pid 40 expected 14 got 10
[rtp @ 0xaaaaf31896c0] PES packet size mismatch
rtp://192.168.0.141:11024: corrupt input packet in stream 0
frame=  124 fps=2.6 q=-0.0 size=  334800kB time=00:00:05.32 bitrate=515406.0kbits/s dup=97 drop=0 speed=0.111x 


  


  What arguments do I provide to ffmpeg and in what order because my stream 0 is metadata and stream 1 is video so as to display image frame by frame with opencv ?
I would be grateful for any help that you could provide.

  


  Further, I also have a query regarding how does ffmpeg know to that it has to first convert the rtp packets into mpeg2 TS packets before segregating video stream and data stream ?

  


• Translating Return To Ringworld

  17 août 2016, par Multimedia Mike — Game Hacking

  As indicated in my previous post, the Translator has expressed interest in applying his hobby towards another DOS adventure game from the mid 1990s : Return to Ringworld (henceforth R2RW) by Tsunami Media. This represents significantly more work than the previous outing, Phantasmagoria.

  
  
  Return to Ringworld Title Screen
  

  I have been largely successful thus far in crafting translation tools. I have pushed the fruits of these labors to a Github repository named improved-spoon (named using Github’s random name generator because I wanted something more interesting than ‘game-hacking-tools’).

  Further, I have recorded everything I have learned about the game’s resource format (named RLB) at the XentaxWiki.

  New Challenges
  The previous project mostly involved scribbling subtitle text on an endless series of video files by leveraging a separate software library which took care of rendering fonts. In contrast, R2RW has at least 30k words of English text contained in various blocks which require translation. Further, the game encodes its own fonts (9 of them) which stubbornly refuse to be useful for rendering text in nearly any other language.

  Thus, the immediate 2 challenges are :
  

  1. Translating volumes of text to Spanish
  2. Expanding the fonts to represent Spanish characters

  Normally, “figuring out the file format data structures involved” is on the list as well. Thankfully, understanding the formats is not a huge challenge since the folks at the ScummVM project already did all the heavy lifting of reverse engineering the file formats.

  The Pitch
  Here was the plan :

  • Create a tool that can dump out the interesting data from the game’s master resource file.
  • Create a tool that can perform the elaborate file copy described in the previous post. The new file should be bit for bit compatible with the original file.
  • Modify the rewriting tool to repack some modified strings into the new resource file.
  • Unpack the fonts and figure out a way to add new characters.
  • Repack the new fonts into the resource file.
  • Repack message strings with Spanish characters.

  Showing The Work : Modifying Strings
  First, I created the tool to unpack blocks of message string resources. I elected to dump the strings to disk as JSON data since it’s easy to write and read JSON using Python, and it’s quick to check if any mistakes have crept in.

  The next step is to find a string to focus on. So I started the game and looked for the first string I could trigger :

  
  
  

  This shows up in the JSON string dump as :

    
      "Spanish" : " !0205Your quarters on the Lance of Truth are spartan, in accord with your mercenary lifestyle.",
      "English" : " !0205Your quarters on the Lance of Truth are spartan, in accord with your mercenary lifestyle."
    ,
  

  As you can see, many of the strings are encoded with an ID key as part of the string which should probably be left unmodified. I changed the Spanish string :

    
      "Spanish" : " !0205Hey, is this thing on ?",
      "English" : " !0205Your quarters on the Lance of Truth are spartan, in accord with your mercenary lifestyle."
    ,
  

  And then I wrote the repacking tool to substitute this message block for the original one. Look ! The engine liked it !

  
  
  

  Little steps, little steps.

  Showing The Work : Modifying Fonts
  The next little step is to find a place to put the new characters. First, a problem definition : The immediate goal is to translate the game into Spanish. The current fonts encoded in the game resource only support 128 characters, corresponding to 7-bit ASCII. In order to properly express Spanish, 16 new characters are required : á, é, í, ó, ú, ü, ñ (each in upper and lower case for a total of 14 characters) as well as the inverted punctuation symbols : ¿, ¡.

  Again, ScummVM already documents (via code) the font coding format. So I quickly determined that each of the 9 fonts is comprised of 128 individual bitmaps with either 1 or 2 bits per pixel. I wrote a tool to unpack each character into an individual portable grey map (PGM) image. These can be edited with graphics editors or with text editors since they are just text files.

  Where to put the 16 new Spanish characters ? ASCII characters 1-31 are non-printable, so my first theory was that these characters would be empty and could be repurposed. However, after dumping and inspecting, I learned that they represent the same set of characters as seen in DOS Code Page 437. So that’s a no-go (so I assumed ; I didn’t check if any existing strings leveraged those characters).

  My next plan was hope that I could extend the font beyond index 127 and use positions 128-143. This worked superbly. This is the new example string :

    
      "Spanish" : " !0205¿Ves esto ? ¡La puntuacion se hace girar !",
      "English" : " !0205Your quarters on the Lance of Truth are spartan, in accord with your mercenary lifestyle."
    ,
  

  Fortunately, JSON understands UTF-8 and after mapping the 16 necessary characters down to the numeric range of 128-143, I repacked the new fonts and the new string :

  
  
  Translation : “See this ? The punctuation is rotated !”
  

  Another victory. Notice that there are no diacritics in this string. None are required for this translation (according to Google Translate). But adding the diacritics to the 14 characters isn’t my department. My tool does help by prepopulating [aeiounAEIOUN] into the right positions to make editing easier for the Translator. But the tool does make the effort to rotate the punctuation since that is easy to automate.

  Next Steps and Residual Weirdness
  There is another method for storing ASCII text inside the R2RW resource called strip resources. These store conversation scripts. There are plenty of fields in the data structures that I don’t fully understand. So, following the lessons I learned from my previous translation outing, I was determined to modify as little as possible. This means copying over most of the original data structures intact, but changing the field representing the relative offset that points to the corresponding string. This works well since the strings are invariably stored NULL-terminated in a concatenated manner.

  I wanted to document for the record that the format that R2RW uses has some weirdness in they way it handles residual bytes in a resource. The variant of the resource format that R2RW uses requires every block to be aligned on a 16-byte boundary. If there is space between the logical end of the resource and the start of the next resource, there are random bytes in that space. This leads me to believe that these bytes were originally recorded from stale/uninitialized memory. This frustrates me because when I write the initial file copy tool which unpacks and repacks each block, I want the new file to be identical to the original. However, these apparent nonsense bytes at the end thwart that effort.

  But leaving those bytes as 0 produces an acceptable resource file.

  Text On Static Images
  There is one last resource type we are working on translating. There are various bits of text that are rendered as images. For example, from the intro :

  
  
  

  It’s possible to locate and extract the exact image that is overlaid on this scene, though without the colors :

  
  
  

  The palettes are stored in a separate resource type. So it seems the challenge is to figure out the palette in use for these frames and render a transparent image that uses the same palette, then repack the new text-image into the new resource file.

  The post Translating Return To Ringworld first appeared on Breaking Eggs And Making Omelettes.

• Why doesn't seem to be able to send an audio file with FRONT_COVER on the Pytelegrambotapi

  27 décembre 2024, par exorik

  In general the problem is that the audio file is sent to a file without a picture. I first thought that the problem is that the picture is installed on the wrong version of id3, and tried four methods of installation

  


  


  1. via the ffmpeg command = [ “ffmpeg”, “-i”, file_path, “-i”, cover_path, “-map”, “0”, “-map”, “1”, “-c:a”, “copy”, “-c:v”, “mjpeg”, “-id3v2_version”, “3“,”-y”, output_file, ]subprocess.run(command, check=True)

     


  


  2. via eyed3, audiofile.tag.images.set(
eyed3.id3.frames.ImageFrame.FRONT_COVER,
cover_data,
“` image/jpeg,
)

     


  


  3. through the mutagen library, tried setting audio.add(
APIC(
encoding=3,
mime=“image/jpeg”,
type=3,
desc=“Cover”,
data=open(saved_photo, “rb”).read(),
)
)
At this stage I realized that the problem is not in the correct id3 tag setting, but in the method through which the audio file is sent. because if I opened it manually and sent it, the cover was there.
But I also tried installing the tag. second version. through the eyed3 library, but that also didn't result in the audio file with the cover art being sent to telegram.

     


  


  


  audiofile w/ out cover

  


  and the exact same audio file, only with the cover art. (it hasn't been altered in any way)

  


  


user_states = {}




def save_audio(message):
    file_info = bot.get_file(message.audio.file_id)
    downloaded_file = bot.download_file(file_info.file_path)

    user_dir = os.path.join("TEMP", str(message.chat.id), "albums")
    os.makedirs(user_dir, exist_ok=True)

    original_file_name = (
        message.audio.file_name
        if message.audio.file_name
        else f"{message.audio.file_id}.mp3"
    )

    file_path = os.path.join(user_dir, original_file_name)

    if message.chat.id not in user_states:
        user_states[message.chat.id] = {"files": [], "stage": None}

    with open(file_path, "wb") as f:
        f.write(downloaded_file)

    user_states[message.chat.id]["files"].append(file_path)

    return file_path


def clear_metadata_for_file(file_path, user_id):
    clear_metadata = settings.get(str(user_id), {}).get("clear", True)

    if clear_metadata:
        temp_file_path = f"{file_path}.temp.mp3"
        command = [
            "ffmpeg",
            "-i",
            file_path,
            "-map_metadata",
            "-1",
            "-c:a",
            "copy",
            "-y",
            temp_file_path,
        ]
        subprocess.run(command, check=True)
        os.replace(temp_file_path, file_path)

        audio_file = eyed3.load(file_path)
        if audio_file.tag is not None:
            audio_file.tag.clear()
        audio_file.tag.save()

    else:
        return file_path


def send_files(message):
    chat_id = message.chat.id
    if chat_id not in user_states or "files" not in user_states[chat_id]:
        return "No files found"

    for file_path in user_states[chat_id]["files"]:
        try:
            with open(file_path, "rb") as f:
                bot.send_audio(chat_id, f)
        except Exception as e:
            return e



@bot.message_handler(content_types=["photo"])
def handle_cover(message):

    if message.content_type == "photo":
        try:

            saved_photo = save_photo(message)


            for file_path in user_states[message.chat.id]["files"]:

                audio = ID3(file_path)
                audio.add(
                    APIC(
                        encoding=3,
                        mime="image/jpeg",
                        type=3,
                        desc="Cover",
                        data=open(saved_photo, "rb").read(),
                    )
                )
                audio.save(file_path)
                with open(saved_photo, "rb") as image_file:
                    image_data = image_file.read()

                audiofile = eyed3.load(file_path)

                if audiofile.tag is None:
                    audiofile.initTag(version=(2, 3, 0))

                audiofile.tag.images.set(
                    3, image_data, "image/jpeg", description="Cover"
                )
                audiofile.tag.save(version=(2, 3, 0))

            send_files(message)

        except Exception as e:
            return

bot.infinity_polling()



  


  I was hoping to get some audio files with cover art, and I'm sure there's no problem with that. But for some reason, the file itself is sent visually without the cover art. I'd really appreciate it if you could help me out with this.