To exchange files, a BitTorrent client needs some information about the exchanged file(s). This information is contained within a file named the “metainfo” file.
More details are explaind in the Metainfo File Structure of the BitTorrent specification web page.
In short, the structure is a UTF-8 bencoded dictionary containing some required and optional key/value pairs.
Example
{ "Announce":"http://.../announce"
"Comment":"..."
"created by":"..."
"creation date":1210177305
"encoding":"UTF-8"
"info":{ "files": [ { "length":3649577
"path":[ "foo.pdf" ]
}
{ "length":47
"path":[ "bar.txt" ]
}
]
"name":"foobar"
"piece Length":65536
"Pieces":"º©g£îèÊÉ\016…\015..."
}
}
Signature
namespace BitTorrent
//---------------------------------------------------------------
// LIBRARIES
//---------------------------------------------------------------
open Common
//---------------------------------------------------------------
// META INFO
//---------------------------------------------------------------
type MetainfoFileEntry =
{ Path : string
Length : int64
MD5 : option<byte[]>
}
type Node = //used for Distributed Hash Table (DHT) protocol
{ IP : string
Port : uint16
}
type Metainfo =
{ mutable Tracker : string //if single tracker
mutable Trackers : string[][] //if multiple trackers
//-----------------------------------------------------------
// Optional
//-----------------------------------------------------------
mutable Comment : string option
mutable CreationDate : int64 option
mutable CreatedBy : string option
//-----------------------------------------------------------
// Base info dictionary
//-----------------------------------------------------------
mutable PieceLength : int64
mutable SHA1s : byte[][]
mutable IsPrivate : bool
//-----------------------------------------------------------
// File info : single and multiple files
//-----------------------------------------------------------
mutable Length : int64 //total length of the download
mutable BaseDirectory : string
mutable Files : MetainfoFileEntry[] //file entries if multiple file
//-----------------------------------------------------------
// Misc
//-----------------------------------------------------------
mutable Infohash : byte[]
//-----------------------------------------------------------
// DHT
//-----------------------------------------------------------
mutable Nodes : list<Node>
}
[<CompilationRepresentation(CompilationRepresentationFlags.ModuleSuffix)>]
module Metainfo =
type t = Metainfo
///[zeroCreate ()] returns a new Metainfo instance.
val zeroCreate : unit -> t
///[fromBValue bvalue] attempts to read [bvalue] into a [Metainfo] structure.
val fromBValue : bvalue:BValue.t -> t
///[fromFile filepath] attempts to read [filepath] into a [Metainfo] structure.
val fromFile : path:string -> t
val ( |SingleTracker|MultipleTrackers| ) : metainfo:t -> Choice<string, (string[][] * string[][])>
///[pieceLength nth metainfo] returns the length of the [nth] piece of the [metainfo].
val pieceLength : piece:int -> metainfo:t -> int64
///[numBlocks nth metainfo] returns the number of blocks in the [nth] piece of the [metainfo].
///The default length of a block is assumed to be [BlockLength].
val numBlocks : piece:int -> metainfo:t -> int
///[blockLength p_th b_th metainfo] returns the length of the [b_th] block of
///the [p_th] piece of the [metainfo].
val blockLength : piece:int -> block:int -> metainfo:t -> int
val toBValue : metainfo:t -> BValue
Implementation
The required helpers are described at the end and those described when we defined bencoding functions.
namespace BitTorrent
//---------------------------------------------------------------
// LIBRARIES
//---------------------------------------------------------------
open System
open System.Collections.Generic
open System.IO
open System.Threading
open Common
open BitTorrent
open BitTorrent.BValue
//---------------------------------------------------------------
// METAINFO
//---------------------------------------------------------------
exception MetainfoException of string
type MetainfoFileEntry =
{ Path : string
Length : int64
MD5 : option<byte[]>
}
type Node =
{ IP : string
Port : uint16
}
type Metainfo =
{ mutable Tracker : string
mutable Trackers : string[][]
//-----------------------------------------------------------
// Optional
//-----------------------------------------------------------
mutable Comment : string option
mutable CreationDate : int64 option
mutable CreatedBy : string option
//-----------------------------------------------------------
// Base info dictionary
//-----------------------------------------------------------
mutable PieceLength : int64
mutable SHA1s : byte[][]
mutable IsPrivate : bool
//-----------------------------------------------------------
// File info : single and multiple files
//-----------------------------------------------------------
mutable Length : int64 //total length of the download
mutable BaseDirectory : string
mutable Files : MetainfoFileEntry[] //file entries if multiple file
//-----------------------------------------------------------
// Misc
//-----------------------------------------------------------
mutable Infohash : byte[]
//-----------------------------------------------------------
// DHT
//-----------------------------------------------------------
mutable Nodes : list<Node>
}
[<CompilationRepresentation(CompilationRepresentationFlags.ModuleSuffix)>]
module Metainfo =
type t = Metainfo
let fail s = raise <| MetainfoException s
let zeroCreate() =
{ Tracker = ""
Trackers = [||]
//-----------------------------------------------------------
// Optional
//-----------------------------------------------------------
Comment = None
CreationDate = None
CreatedBy = None
//-----------------------------------------------------------
// Base info dictionary
//-----------------------------------------------------------
PieceLength = 0L
SHA1s = [||]
IsPrivate = false
//-----------------------------------------------------------
// File info : single and multiple files
//-----------------------------------------------------------
Length = 0L
BaseDirectory = ""
Files = [||]
//-----------------------------------------------------------
// Misc
//-----------------------------------------------------------
Infohash = [||]
//-----------------------------------------------------------
// DHT
//-----------------------------------------------------------
Nodes = []
}
let fromBValue bvalue =
let metainfo = zeroCreate()
//
// Compulsory fields
//
match tryGetValue "announce" bvalue with
| Some announce ->
let announceUrl = getString announce
if announceUrl.StartsWith "http" then metainfo.Tracker <- announceUrl
| None -> fail "Missing [announce] key"
//we need to keep track of the info dictionary
let mutable info = Unchecked.defaultof<BValue>
//temporary extract to make sure the bvalue contains an info dictionary
match tryGetValue "info" bvalue with
| Some res ->
info <- res
metainfo.Infohash <- info |> BValue.toBytes |> Digest.SHA1.hexdigest
| None -> fail "Missing [info] key"
//
// Info dictionary parsing
//
// Common fields
match tryGetValue "piece length" info with
| Some pieceLength -> metainfo.PieceLength <- getInt64 pieceLength
| None -> fail "Missing [piece length] key in [info] dictionary"
match tryGetValue "pieces" info with
| Some pieces ->
let pieces = getBytes pieces
let piecesCount = Array.length pieces
if piecesCount % 20 <> 0 then
fail "Invalid [pieces] value in info dictionary"
metainfo.SHA1s <- [|for i in 0 .. 20 .. piecesCount - 1 -> pieces.[i .. i + 19] |]
| None -> fail "Missing [pieces] key in [info] dictionary"
let mutable nameOrDirectory = ""
match tryGetValue "name" info with
| Some name -> nameOrDirectory <- getString name
| None -> fail "Missing [name] key in [info] dictionary"
//optional field used for single file download
match tryGetValue "length" info with
//single file
| Some length ->
metainfo.Length <- getInt64 length
let md5 =
match tryGetValue "md5sum" info with
| Some md5 -> Some <| getBytes md5
| None -> None
let entry =
{ Path = nameOrDirectory
Length = metainfo.Length
MD5 = md5
}
metainfo.Files <- [|entry|]
//multiple files
| None ->
match tryGetValue "files" info with
| Some files ->
//with multiple files, the "name" key refers to the base directory
metainfo.BaseDirectory <- nameOrDirectory
let entries =
[|let length = ref 0L
let fullpath = ref ""
let md5sum : option<byte[]> ref = ref None
for file in getList files do
match tryGetValue "length" file with
| Some len -> length := getInt64 len
| None -> fail "Missing [length] key in [files] value of [info] dictionary"
//the full path is given as a list of folders
fullpath := ""
match tryGetValue "path" file with
| Some pathComponents ->
for subdir in getList pathComponents do
fullpath := Filename.concat !fullpath (getString subdir)
| None -> fail "Missing [path] key in [files] value of [info] dictionary"
//md5 is optional and not used
md5sum := None
tryGetValue "md5sum" file |> Option.iter (fun md5 ->
md5sum := Some <| getBytes md5
)
//aggregate the files lengths to compute the transfer length
metainfo.Length <- metainfo.Length + !length
let metainfoFile =
{ Path = !fullpath
Length = !length
MD5 = !md5sum
}
yield metainfoFile
|]
metainfo.Files <- entries
| None -> fail "Missing [files] or [length] key in [info] dictionary"
//
// Optional fields from the info dictionary
//
//used for Distributed Hash Tables (DHT)
tryGetValue "private" info |> Option.iter (fun isPrivate ->
metainfo.IsPrivate <- getInt64 isPrivate = 1L
)
//used for Distributed Hash Tables (DHT)
tryGetValue "nodes" info |> Option.iter (fun nodes ->
let nodes =
[ for node in getList nodes do
let nodeData = getList node
if nodeData.Count <> 2 then
fail "Invalid [nodes] value in the [info] BDict"
let node =
{ IP = getString nodeData.[0]
Port = nodeData.[1] |> getInt64 |> uint16
}
yield node
]
metainfo.Nodes <- nodes
)
//
// Optional fields from the main dictionary
//
//Note : when announce-list is present, the "announce" value is overriden
tryGetValue "announce-list" bvalue |> Option.iter (fun announceList ->
let tiers = getList announceList
metainfo.Trackers <- Array.zeroCreate tiers.Count
let tierIdx = ref 0
let randomizedTrackers =
Array.init tiers.Count (fun i ->
[|for url in getList tiers.[i] do
let surl = getString url
yield surl
|]
|> Array.shuffled
) |> Array.filter (fun tier -> tier.Length > 0)
metainfo.Trackers <- randomizedTrackers
)
tryGetValue "comment" bvalue |> Option.iter (fun comment ->
metainfo.Comment <- comment |> getString |> Some
)
tryGetValue "creation date" bvalue |> Option.iter (fun creationDate ->
metainfo.CreationDate <- creationDate |> getInt64 |> Some
)
tryGetValue "created by" bvalue |> Option.iter (fun createdBy ->
metainfo.CreatedBy <- createdBy |> getString |> Some
)
metainfo
let fromFile path =
path |> BValue.fromFile |> fromBValue
let ( |SingleTracker|MultipleTrackers| ) metainfo =
if metainfo.Trackers.Length = 0 then
SingleTracker metainfo.Tracker
else
let http =
[|for tier in metainfo.Trackers do
let xs = tier |> Array.filter (fun url -> url.StartsWith "http://")
if xs.Length > 0 then yield xs
|]
let udp =
[|for tier in metainfo.Trackers do
let xs = tier |> Array.filter (fun url -> url.StartsWith "udp://")
if xs.Length > 0 then yield xs
|]
MultipleTrackers (http, udp)
let isLastPiece piece metainfo =
piece = metainfo.SHA1s.Length - 1
let pieceLength piece metainfo =
if piece < 0 || piece >= metainfo.SHA1s.Length then
let reason = sprintf "piece %d is out of bounds" piece
invalidArg "piece" reason
if isLastPiece piece metainfo then
metainfo.Length % metainfo.PieceLength
else
metainfo.PieceLength
let numBlocks piece metainfo =
let pieceLength = pieceLength piece metainfo
let blockLength = int64 App.BlockLength
let nBlocks = pieceLength / blockLength |>int
if pieceLength % blockLength = 0L then nBlocks else nBlocks + 1
let isLastBlock piece block metainfo =
let nBlocks = numBlocks piece metainfo
isLastPiece piece metainfo && block = nBlocks - 1
let blockLength piece block metainfo =
let nBlocks = numBlocks piece metainfo
if block >= nBlocks then
let reason = sprintf "block %d is out of bounds" block
invalidArg "block" reason
if isLastBlock piece block metainfo then
metainfo.Length % int64 App.BlockLength |> int
else
App.BlockLength
//-------------------------------------------------------------
// Metainfo creation
//-------------------------------------------------------------
let announceListToBValue announceList =
let tiers = List<_>()
for tier in announceList do
let btier = List<_>()
for tracker in tier do
btier.Add(BString (String.toBytes tracker))
tiers.Add(BList btier)
BList tiers
let sha1sToBValue (sha1s:byte[][]) =
BString [|for sha1 in sha1s do yield! sha1|]
let pathToBValue (path:string) =
let elements = List<_>()
path.Split([|Path.DirectorySeparatorChar|], StringSplitOptions.RemoveEmptyEntries)
|> Array.iter (fun elt -> String.toBytes elt |> BString |> elements.Add)
BList elements
let nodesToBValue (nodes:list<Node>) =
let xs = List<_>()
nodes |> List.iter (fun node ->
let bnode = List<_>()
bnode.Add (BString (String.toBytes node.IP))
bnode.Add (BInt (int64 node.Port))
xs.Add(BList bnode)
)
BList xs
let toBValue metainfo =
let dict = new SortedDictionary<_, _>()
dict.Add("announce", BString (String.toBytes metainfo.Tracker))
dict.Add("announce-list", (announceListToBValue metainfo.Trackers))
metainfo.Comment |> Option.iter (fun comment ->
dict.Add("comment", BString (String.toBytes comment))
)
metainfo.CreationDate |> Option.iter (fun date ->
dict.Add("creation date", BInt date)
)
metainfo.CreatedBy |> Option.iter (fun createdBy ->
dict.Add("created by", BString (String.toBytes createdBy))
)
let info = new SortedDictionary<_, _>()
info.Add("piece length", BInt metainfo.PieceLength)
info.Add("pieces", sha1sToBValue metainfo.SHA1s)
info.Add("private", BInt (if metainfo.IsPrivate then 1L else 0L))
info.Add("nodes", nodesToBValue metainfo.Nodes)
let isSingleFile = metainfo.Files.Length = 1
if isSingleFile then
info.Add("name", BString (String.toBytes metainfo.Files.[0].Path))
info.Add("length", BInt metainfo.Files.[0].Length)
else
info.Add("name", BString (String.toBytes metainfo.BaseDirectory))
let files = List<_>()
metainfo.Files |> Array.iter (fun file ->
let bfile = new SortedDictionary<_, _>()
bfile.Add("length", BInt file.Length)
bfile.Add("path", pathToBValue file.Path)
files.Add(BDict bfile)
)
info.Add("files", BList files)
dict.Add("info", BDict info)
BDict dict
Helpers
namespace Common
open System
module Array =
let random = new Random(DateTime.Now.Ticks >>> 32 |> int)
let swap n n' data =
let max = Array.length data
if n' >= max || n' < 0 then
invalidArg "n'" <| sprintf "swap n:%d n':%d data:%A : index was out of array bounds" n n' data
if n >= max || n < 0 then
invalidArg "n'" <| sprintf "swap n:%d n':%d data:%A : index was out of array bounds" n n' data
let tmp = data.[n]
data.[n] <- data.[n']
data.[n'] <- tmp
let shuffle data =
data |> Array.iteri (fun i _ -> data |> swap i (random.Next(data.Length)))
let shuffled data =
shuffle data
data
//=========================================================================
open System.IO
open System.Security.Cryptography
module Digest =
module SHA1 =
let hexdigest (bytes : byte[]) =
let csp = new SHA1CryptoServiceProvider()
csp.ComputeHash(bytes)
let digestFile path =
if File.Exists(path) then
let csp = new SHA1CryptoServiceProvider()
use s = File.OpenRead(path)
csp.ComputeHash(s :> Stream)
else
invalidArg "file" "file not found"