Simplify packet processing. Re-enable pkt processing tests.

This commit is contained in:
Deon George 2023-12-13 23:00:47 +11:00
parent 26c80dc1c5
commit aae551aacf
12 changed files with 255 additions and 163 deletions

View File

@ -565,7 +565,7 @@ class Message extends FTNBase
$o->header = unpack(self::unpackheader(self::header),substr($msg,0,self::HEADER_LEN));
} catch (\Exception $e) {
Log::error(sprintf('%s:! Error bad packet header',self::LOGKEY));
Log::error(sprintf('%s:! Error bad packet header',self::LOGKEY),['e'=>$e->getMessage(),'header'=>substr($msg,0,self::HEADER_LEN)]);
$validator = Validator::make([
'header' => substr($msg,0,self::HEADER_LEN),
],[

View File

@ -22,8 +22,8 @@ class Packet extends FTNBase implements \Iterator, \Countable
{
private const LOGKEY = 'PKT';
private const BLOCKSIZE = 1024;
protected const PACKED_MSG_LEAD = "\02\00";
protected const PACKED_END = "\00\00";
public const regex = '([[:xdigit:]]{4})(?:-(\d{4,10}))?-(.+)';
@ -172,7 +172,7 @@ class Packet extends FTNBase implements \Iterator, \Countable
/**
* Process a packet file
*
* @param mixed $f
* @param mixed $f File handler returning packet data
* @param string $name
* @param int $size
* @param Domain|null $domain
@ -213,18 +213,18 @@ class Packet extends FTNBase implements \Iterator, \Countable
$o->name = $name;
$x = fread($f,2);
if (strlen($x) === 2) {
// End of Packet?
if ($x === "\00\00")
return $o;
// End of Packet?
if ((strlen($x) === 2) && ($x === "\00\00"))
return $o;
// Messages start with self::PACKED_MSG_LEAD
if ((strlen($x) === 2) && ($x !== self::PACKED_MSG_LEAD))
throw new InvalidPacketException('Not a valid packet: '.bin2hex($x));
// Messages start with self::PACKED_MSG_LEAD
elseif ($x !== self::PACKED_MSG_LEAD)
throw new InvalidPacketException('Not a valid packet: '.bin2hex($x));
// No message attached
else if (! strlen($x))
throw new InvalidPacketException('No message in packet: '.bin2hex($x));
} else
throw new InvalidPacketException('Not a valid packet, not EOP or SOM'.bin2hex($x));
// Work out the packet zone
if ($o->fz && ($o->fd || $domain)) {
@ -250,98 +250,36 @@ class Packet extends FTNBase implements \Iterator, \Countable
->singleOrFail();
}
$buf_ptr = 0; // Pointer to the end of the current message
$message = ''; // Current message we are building
$readbuf = ''; // What we are reading to determine the message contents
$last = ''; // If during buffer reads, the end of the last buffer had our NULL end of message marker
$msgbuf = '';
$leader = Message::HEADER_LEN+strlen(self::PACKED_MSG_LEAD);
// We loop through reading from the buffer, to find our end of message tag
while ($buf_ptr || (! feof($f) && ($readbuf=fread($f,self::BLOCKSIZE)))) {
if (! $buf_ptr)
$read_ptr = ftell($f);
while ((! feof($f) && ($readbuf=fread($f,$leader)))) {
$read_ptr = ftell($f);
$msgbuf .= $readbuf;
// Packed messages are Message::HEADER_LEN, prefixed with self::PACKED_MSG_LEAD
// If we havent got our header yet
if (strlen($message) < (Message::HEADER_LEN+strlen(self::PACKED_MSG_LEAD))) {
$addchars = (Message::HEADER_LEN+strlen(self::PACKED_MSG_LEAD))-strlen($message);
$message .= substr($readbuf,$buf_ptr,$addchars);
$buf_ptr += $addchars;
// See if we have our EOM/EOP marker
if ((($end=strpos($msgbuf,"\x00".self::PACKED_MSG_LEAD,$leader)) !== FALSE)
|| (($end=strpos($msgbuf,"\x00".self::PACKED_END,$leader)) !== FALSE))
{
// Parse our message
$o->parseMessage(substr($msgbuf,0,$end));
// If our buffer wasnt big enough, and thus $addchars didnt have enough chars to add.
if ($buf_ptr >= strlen($readbuf)) {
$buf_ptr = 0;
continue;
}
$msgbuf = substr($msgbuf,$end+3);
continue;
// If we have more to read
} elseif ($read_ptr < $size) {
continue;
}
// Take 2 chars from the buffer and check if we have our end packet signature
// $last is set, because we detected a NULL, so we'll add two more chars and see if we have our EOM
// signature. Some of those chars may belong to the next message, if $last has more than 1 NULL.
if ($last && ($buf_ptr === 0)) {
$last .= substr($readbuf,0,2);
if (($end=strpos($last,"\x00".self::PACKED_MSG_LEAD,$buf_ptr)) !== FALSE) {
$o->parseMessage(substr($message,0,$end-2));
$last = '';
$message = '';
$buf_ptr = 1+$end;
// Loop to rebuild our header for the next message
continue;
}
// We didnt have an EOM marker
$last = '';
}
// See if our EOM marker is in the read buffer
if (($end=strpos($readbuf,"\x00".self::PACKED_MSG_LEAD,$buf_ptr)) === FALSE) {
// Just in case our packet break is at the end of the buffer
$last = substr($readbuf,-2);
// We have an EOM or EOP marker here, so loop around to get the next read
if (str_contains($last,"\x00") && ($size < $read_ptr)) {
$message .= substr($readbuf,$buf_ptr);
$buf_ptr = 0;
continue;
}
// No EOM marker
$last = '';
// See if we have an EOP marker
$end = strpos($readbuf,"\x00\x00\x00",$buf_ptr);
}
// See if we have found the end of the packet, if not read more.
if ($end === FALSE) {
if ($read_ptr < $size) {
$message .= substr($readbuf,$buf_ptr);
$buf_ptr = 0;
continue;
// No more to read, so the packet is bad
} else
throw new InvalidPacketException(sprintf('Cannot determine END of message/packet: %s|%s',get_class($o),hex_dump($message)));
} else {
$message .= substr($readbuf,$buf_ptr,$end-$buf_ptr);
$buf_ptr = $end+3;
if ($buf_ptr >= strlen($readbuf))
$buf_ptr = 0;
}
// Look for the next message
$o->parseMessage($message);
$message = '';
// If we get here
throw new InvalidPacketException(sprintf('Cannot determine END of message/packet: %s|%s',get_class($o),hex_dump($message)));;
}
// If our message is still set, then we have an unprocessed message
if ($message)
$o->parseMessage($message);
if ($msgbuf)
throw new InvalidPacketException(sprintf('Unprocessed data in packet: %s|%s',get_class($o),hex_dump($msgbuf)));
return $o;
}
@ -546,4 +484,9 @@ class Packet extends FTNBase implements \Iterator, \Countable
$this->messages->push($msg);
}
public function pluck(string $key): Collection
{
return $this->messages->pluck($key);
}
}

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -2,132 +2,279 @@
namespace Tests\Feature;
use Illuminate\Foundation\Testing\DatabaseTransactions;
use Illuminate\Support\Facades\Storage;
use Tests\TestCase;
use App\Classes\File;
use App\Classes\FTN\Packet;
use App\Classes\FTN\{Message,Packet};
use App\Models\{Address,Domain,System,Zone};
class PacketTest extends TestCase
{
private System $so;
use DatabaseTransactions;
//private System $so;
private Domain $do;
private function init()
private function init_fidonet(): void
{
System::unguard();
Domain::unguard();
$this->so = System::firstOrCreate(['name'=>'test','sysop'=>'sysop','location'=>'location','active'=>TRUE]);
$this->do = Domain::firstOrCreate(['name'=>'packets','active'=>TRUE]);
$do = new Domain;
$do->active = TRUE;
$do->name = 'fidonet';
$do->flatten = TRUE;
$do->save();
$zo = new Zone;
$zo->zone_id = 1;
$zo->active = TRUE;
$zo->system_id = 2;
$zo->default = TRUE;
$do->zones()->save($zo);
$zo = new Zone;
$zo->zone_id = 2;
$zo->active = TRUE;
$zo->system_id = 3;
$zo->default = TRUE;
$do->zones()->save($zo);
$zo = new Zone;
$zo->zone_id = 3;
$zo->active = TRUE;
$zo->system_id = 1;
$zo->default = TRUE;
$do->zones()->save($zo);
Address::createFTN('3:633/280@fidonet',System::findOrFail(1));
Address::createFTN('3:633/2744@fidonet',System::findOrFail(2));
$this->do = $do;
}
public function test_null()
private function init_fsxnet(): void
{
return $this->assertTrue(TRUE);
$do = new Domain;
$do->active = TRUE;
$do->name = 'fsxnet';
$do->flatten = TRUE;
$do->save();
$zo = new Zone;
$zo->zone_id = 21;
$zo->active = TRUE;
$zo->system_id = 1;
$zo->default = TRUE;
$do->zones()->save($zo);
Address::createFTN('21:3/100@fsxnet',System::findOrFail(2));
Address::createFTN('21:3/2744@fsxnet',System::findOrFail(3));
Address::createFTN('21:3/184@fsxnet',System::findOrFail(4));
$this->do = $do;
}
private function init_private(): void
{
$do = new Domain;
$do->active = TRUE;
$do->name = 'private';
$do->flatten = TRUE;
$do->save();
$zo = new Zone;
$zo->zone_id = 10;
$zo->active = TRUE;
$zo->system_id = 1;
$zo->default = TRUE;
$do->zones()->save($zo);
Address::createFTN('10:1/1@private',System::findOrFail(3));
Address::createFTN('10:1/2@private',System::findOrFail(4));
$this->do = $do;
}
private function mail_file(string $file)
{
$fs = Storage::disk(config('fido.local_disk'));
return new File($fs->path(sprintf('%s/%s',config('fido.dir'),$file)));
}
public function test_packet_1()
{
$this->init_fidonet();
$f = $this->mail_file('mail/018A-1702393055-78754407.pkt');
foreach ($f as $packet) {
$pkt = Packet::process($packet,$x=$f->itemName(),$f->itemSize(),$this->do);
$this->assertInstanceOf(Packet\FSC39::class,$pkt);
$this->assertEquals('ABCDEFGH',$pkt->password);
$this->assertEquals(1,count($pkt));
$this->assertEquals('3634/27 12 154/10 221/6 218/840 770/1 633/280',$pkt->messages[0]->path[0]);
$this->assertEquals('1/120 18/0 116/116 120/616 123/0 10 25 126 160 180 200 525 755 3001',$pkt->messages[0]->seenby[0]);
$this->assertEquals('*The Gate BBS*Shelby, NC USA*thegateb.synchro.net* (1:3634/27)',$pkt->messages[0]->origin);
$this->assertEquals('1:3634/27.0@fidonet',$pkt->messages[0]->fftn);
$this->assertEquals('1:633/2744.0@fidonet',$pkt->messages[0]->tftn);
$this->assertEquals('Gate Keeper',$pkt->messages[0]->user_from);
$this->assertEquals('Meitsi',$pkt->messages[0]->user_to);
$this->assertEquals('Status of HAM radio in FidoNet',$pkt->messages[0]->subject);
$this->assertEquals('275.fidonet_ham@1:3634/27 29ddab74',$pkt->messages[0]->msgid);
$this->assertEquals('1:229/428 012c0322',$pkt->messages[0]->replyid);
$this->assertEquals('bc291b3ea15750c2d0c39c9221093901',md5($pkt->messages[0]->message));
}
}
public function test_packet_2()
{
$this->init_fidonet();
$f = $this->mail_file('mail/018A-1701955391-71c7a400.pkt');
foreach ($f as $packet) {
$pkt = Packet::process($packet,$x=$f->itemName(),$f->itemSize(),$this->do);
$this->assertInstanceOf(Packet\FSC39::class,$pkt);
$this->assertEquals('ABCDEFGH',$pkt->password);
$this->assertEquals(43,count($pkt));
$this->assertEquals('229/664 426 633/280',$pkt->messages[0]->path[0]);
$this->assertEquals('15/0 106/201 128/260 129/305 153/7715 218/700 221/6 226/30 227/114',$pkt->messages[0]->seenby[0]);
$this->assertEquals('Northern Realms (1:229/664)',$pkt->messages[0]->origin);
$this->assertEquals('1:229/664.0@fidonet',$pkt->messages[0]->fftn);
$this->assertEquals('1:633/2744.0@fidonet',$pkt->messages[0]->tftn);
$this->assertEquals('Northern Realms',$pkt->messages[0]->user_from);
$this->assertEquals('All',$pkt->messages[0]->user_to);
$this->assertEquals('NYPost Daily Horoscope',$pkt->messages[0]->subject);
$this->assertEquals('1:229/664 56db6f89',$pkt->messages[0]->msgid);
$this->assertEquals('',$pkt->messages[0]->replyid);
$this->assertEquals('b8c33987647e88f86456f0e571e98398',md5($pkt->messages[0]->message));
}
}
public function test_packet_3()
{
$this->init_fsxnet();
$f = $this->mail_file('mail/0B39-1701919239-65713a06.pkt');
foreach ($f as $packet) {
$pkt = Packet::process($packet,$x=$f->itemName(),$f->itemSize(),$this->do);
$this->assertInstanceOf(Packet\FSC48::class,$pkt);
$this->assertEquals('ABCDEFG#',$pkt->password);
$this->assertEquals(4,count($pkt));
$this->assertInstanceOf(Message::class,$pkt->messages[0]);
$this->assertEquals('3/165 100 184',$pkt->messages[0]->path[0]);
$this->assertEquals('1/100 179 2/100 116 3/100 105 107 108 109 110 111 113 119 120 126 127',$pkt->messages[0]->seenby[0]);
$this->assertEquals('www.theunderground.us Telnet 10023 SSH 7771 (21:3/165)',$pkt->messages[0]->origin);
$this->assertEquals('21:3/165.0@fsxnet',$pkt->messages[0]->fftn);
$this->assertEquals('21:3/2744.0@fsxnet',$pkt->messages[0]->tftn);
$this->assertEquals('ibbslastcall',$pkt->messages[0]->user_from);
$this->assertEquals('All',$pkt->messages[0]->user_to);
$this->assertEquals('ibbslastcall-data',$pkt->messages[0]->subject);
$this->assertEquals('21:3/165 6b42fe09',$pkt->messages[0]->msgid);
$this->assertEquals('',$pkt->messages[0]->replyid);
$this->assertEquals('aa6bc82b63355ab68889f61822305072',md5($pkt->messages[0]->message));
}
}
/*
* @todo This packet doesnt have an origin line, need a new one with nomsg and an origin line
public function test_nomsgid_origin()
{
$this->init();
$this->init_private();
Zone::unguard();
Address::unguard();
$zo = Zone::firstOrCreate(['zone_id'=>21,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]);
$src = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>3,'node_id'=>151,'point_id'=>0,'role'=>Address::NODE_ACTIVE,'active'=>TRUE,'system_id'=>$this->so->id]);
$hub = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>1,'node_id'=>1,'point_id'=>0,'role'=>Address::NODE_ACTIVE,'active'=>TRUE,'system_id'=>$this->so->id]);
$ao = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>1,'node_id'=>4,'point_id'=>0,'role'=>Address::NODE_ACTIVE,'active'=>TRUE,'system_id'=>$this->so->id]);
// This packet has an incorrect zone in the Origin
$f = new File(__DIR__.'/data/test_nomsgid_origin.pkt');
$f = $this->mail_file('mail/test_nomsgid_origin.pkt');
foreach ($f as $packet) {
$pkt = Packet::process($packet,$f->itemName(),$f->itemSize());
$this->assertInstanceOf(Packet\FSC39::class,$pkt);
$this->assertEquals(1,$pkt->count());
$messages = FALSE;
foreach ($pkt as $msg) {
$messages = TRUE;
$this->assertNotTrue($msg->isNetmail());
$this->assertNotFalse($msg->pathaddress->search($hub->id));
$this->assertCount(0,$msg->rogue_seenby);
$this->assertEquals('PVT_TEST',$msg->echoarea);
}
$this->assertTrue($messages);
$this->assertInstanceOf(Message::class,$pkt->messages[0]);
$this->assertEquals('1/1 999/1',$pkt->messages[0]->path[0]);
$this->assertEquals('Daytona BBS (Netherlands) (10:3/151)',$pkt->messages[0]->origin);
$this->assertEquals('10:3/151.0@private',$pkt->messages[0]->fftn);
$this->assertEquals('10:999/999.0@private',$pkt->messages[0]->tftn);
$this->assertEquals('Hub Robot',$pkt->messages[0]->user_from);
$this->assertEquals('Henk Hilgersum',$pkt->messages[0]->user_to);
$this->assertEquals('LEMEIN',$pkt->messages[0]->subject);
$this->assertEquals('',$pkt->messages[0]->msgid);
$this->assertEquals('',$pkt->messages[0]->replyid);
$this->assertEquals('6bbe3d0475cf60804ca4e942212d3456',md5($pkt->messages[0]->message));
}
}
*/
/*
* @todo We are not correctly setup to parse messages without an origin line
public function test_nomsgid_noorigin()
{
$this->init();
Zone::unguard();
Address::unguard();
$zo = Zone::firstOrCreate(['zone_id'=>10,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]);
$src = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>999,'node_id'=>1,'point_id'=>0,'role'=>Address::NODE_ACTIVE,'active'=>TRUE,'system_id'=>$this->so->id]);
$this->init_private();
// This packet has no Origin Line
$f = new File(__DIR__.'/data/test_nomsgid_noorigin.pkt');
$f = $this->mail_file('mail/test_nomsgid_noorigin.pkt');
foreach ($f as $packet) {
$pkt = Packet::process($packet,$f->itemName(),$f->itemSize(),$this->do);
$this->assertInstanceOf(Packet\FSC39::class,$pkt);
$this->assertEquals(1,$pkt->count());
$messages = FALSE;
foreach ($pkt as $msg) {
$messages = TRUE;
dump($msg);
$this->assertNotTrue($msg->isNetmail());
$this->assertNotFalse($msg->path->search('1/1 999/1'));
$this->assertNotFalse($msg->seenby->search('1/1 4 3/0 999/1 999'));
}
$this->assertTrue($messages);
}
}
*/
/*
public function test_msgid_origin()
{
$this->init();
Zone::unguard();
Address::unguard();
$zo = Zone::firstOrCreate(['zone_id'=>10,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]);
$src = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>999,'node_id'=>1,'point_id'=>0,'role'=>Address::NODE_ACTIVE,'active'=>TRUE,'system_id'=>$this->so->id]);
$this->init_private();
// This packet has an incorrect zone in the Origin
$f = new File(__DIR__.'/data/test_msgid_origin.pkt');
$f = $this->mail_file('mail/test_msgid_origin.pkt');
foreach ($f as $packet) {
$pkt = Packet::process($packet,$f->itemName(),$f->itemSize(),$this->do);
$this->assertInstanceOf(Packet\FSC39::class,$pkt);
$this->assertEquals(1,$pkt->count());
$messages = FALSE;
foreach ($pkt as $msg) {
$messages = TRUE;
$this->assertNotTrue($msg->isNetmail());
$this->assertEquals('PVT_TEST',$msg->echoarea);
$this->assertNotFalse($msg->seenby->search('1/1 999/1 999'));
}
$this->assertTrue($messages);
$this->assertInstanceOf(Message::class,$pkt->messages[0]);
$this->assertEquals('999/1',$pkt->messages[0]->path[0]);
$this->assertEquals('Alterant MailHUB at your service (10:999/1)',$pkt->messages[0]->origin);
$this->assertEquals('10:999/1.0@private',$pkt->messages[0]->fftn);
$this->assertEquals('10:999/999.0@private',$pkt->messages[0]->tftn);
$this->assertEquals('Hub Robot',$pkt->messages[0]->user_from);
$this->assertEquals('deon',$pkt->messages[0]->user_to);
$this->assertEquals('Hub Robot Report',$pkt->messages[0]->subject);
$this->assertEquals('10:999/1 612aecda',$pkt->messages[0]->msgid);
$this->assertEquals('',$pkt->messages[0]->replyid);
$this->assertEquals('61078e680cda04c8b5eba0f712582e70',md5($pkt->messages[0]->message));
}
}
public function test_packet_parse()
public function test_soh_char_soa()
{
$this->init();
$zo = Zone::firstOrCreate(['zone_id'=>21,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]);
$this->init_fsxnet();
// This packet has a SOH<char>SOH sequence
$f = new File(__DIR__.'/data/test_binary_content-2.pkt');
$f = $this->mail_file('mail/test_binary_content-2.pkt');
foreach ($f as $packet) {
$pkt = Packet::process($packet,$f->itemName(),$f->itemSize(),$zo->domain);
$pkt = Packet::process($packet,$f->itemName(),$f->itemSize(),$this->do);
$this->assertEquals(1,$pkt->count());
@ -147,9 +294,14 @@ class PacketTest extends TestCase
$this->assertTrue($messages);
}
}
public function test_soh_message()
{
$this->init_fsxnet();
// This packet has SOH in the message content
$f = new File(__DIR__.'/data/test_binary_content.pkt');
$f = $this->mail_file('mail/test_binary_content.pkt');
foreach ($f as $packet) {
$pkt = Packet::process($packet,$f->itemName(),$f->itemSize());
@ -171,9 +323,14 @@ class PacketTest extends TestCase
$this->assertTrue($messages);
}
}
public function test_bad_zone()
{
$this->init_private();
// This packet has an incorrect zone in the Origin
$f = new File(__DIR__.'/data/test_msgid_origin.pkt');
$f = $this->mail_file('mail/test_msgid_origin.pkt');
foreach ($f as $packet) {
$pkt = Packet::process($packet,$f->itemName(),$f->itemSize());
@ -199,17 +356,10 @@ class PacketTest extends TestCase
public function test_soh_in_origin()
{
$this->init();
Zone::unguard();
Address::unguard();
$zo = Zone::firstOrCreate(['zone_id'=>3,'default'=>TRUE,'active'=>TRUE,'domain_id'=>$this->do->id,'system_id'=>$this->so->id]);
$src = Address::firstOrCreate(['zone_id'=>$zo->id,'region_id'=>0,'host_id'=>712,'node_id'=>886,'point_id'=>0,'role'=>Address::NODE_ACTIVE,'active'=>TRUE,'system_id'=>$this->so->id]);
$this->init_fidonet();
// This packet has a SOH<char>SOH sequence
$f = new File(__DIR__.'/data/test_msg_with_soh_in_origin.pkt');
$f = $this->mail_file('mail/test_msg_with_soh_in_origin.pkt');
foreach ($f as $packet) {
$pkt = Packet::process($packet,$f->itemName(),$f->itemSize());
@ -251,5 +401,4 @@ class PacketTest extends TestCase
$this->assertTrue($messages);
}
}
*/
}
}