[Bps-public-commit] rt-extension-aws-assets branch sync-utility updated. 2e43ca30bb94c04ea71287d19567fbb5451896df
BPS Git Server
git at git.bestpractical.com
Wed Feb 14 21:54:11 UTC 2024
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "rt-extension-aws-assets".
The branch, sync-utility has been updated
via 2e43ca30bb94c04ea71287d19567fbb5451896df (commit)
via aafe093b88c955ea0d05c4075af281396de3b119 (commit)
via 52c84c1ce5c939250f8114653e0cfc0e3ae0287d (commit)
via 68f7dfb8d1406bf93119772832649675463821d9 (commit)
via 98a69c5c092821751e21f41eccbb8e78f59a52cd (commit)
via 7c2801533ae32e791b1a456e54c2fe955522531a (commit)
via f93a5df87a629cf5e31ffa1798bd9e1f4276e2a2 (commit)
via b7ae20b5fecb3903c48d604b5282122c128519ab (commit)
via e9a4aef303e4008fb6e6fd0b691599f28ad6a636 (commit)
from a2841594f394d5965e109278592c43533fbbacce (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 2e43ca30bb94c04ea71287d19567fbb5451896df
Author: Jim Brandt <jbrandt at bestpractical.com>
Date: Wed Feb 14 15:54:44 2024 -0500
Use the correct argument name for the token
The API parameter is Marker, but the argument in the method
is Token. The previous code didn't find the token and would
try to fetch the same set of resources in an endless loop.
diff --git a/lib/RT/Extension/AWS/Assets.pm b/lib/RT/Extension/AWS/Assets.pm
index ca52320..4f6cedf 100644
--- a/lib/RT/Extension/AWS/Assets.pm
+++ b/lib/RT/Extension/AWS/Assets.pm
@@ -214,7 +214,7 @@ sub FetchMultipleAssetsFromAWS {
eval {
my $service = Paws->service($args{'ServiceType'}, credentials => $credentials, region => $args{'Region'});
- if ( $args{'Marker'} ) {
+ if ( $args{'Token'} ) {
$res = $service->DescribeDBInstances(MaxRecords => $args{'MaxResults'}, Marker => $args{'Token'});
}
else {
commit aafe093b88c955ea0d05c4075af281396de3b119
Author: Jim Brandt <jbrandt at bestpractical.com>
Date: Wed Feb 14 15:52:36 2024 -0500
Use the Init from RTs CLI to enable debug
diff --git a/bin/rt-import-aws-assets.in b/bin/rt-import-aws-assets.in
index 6f2dbcd..9f3e5cc 100644
--- a/bin/rt-import-aws-assets.in
+++ b/bin/rt-import-aws-assets.in
@@ -10,12 +10,9 @@ BEGIN {
### after: use lib qw(@RT_LIB_PATH@);
use lib '/opt/rt5/local/lib /opt/rt5/lib';
use RT;
-RT::LoadConfig();
-RT::Init();
}
use RT::Interface::CLI qw(GetCurrentUser loc);
-use RT::Extension::AWS::Assets;
__PACKAGE__->run(@ARGV) unless caller;
@@ -23,6 +20,7 @@ sub run{
my ($class, @args) = @_;
my %args = $class->process_args(@args);
+ require RT::Extension::AWS::Assets;
if ( $args{insert} ) {
my ($aws_resources, $token);
@@ -76,7 +74,7 @@ sub process_args {
local @ARGV = @_;
my %opt;
- Getopt::Long::GetOptions( \%opt, 'help|h', 'insert|i', 'update|u', 'catalog=s', 'type=s', 'reserved', 'region=s', 'results=i', 'debug|d' );
+ RT::Interface::CLI::Init( \%opt, 'help|h', 'insert|i', 'update|u', 'catalog=s', 'type=s', 'reserved', 'region=s', 'results=i', 'debug|d' );
if ( delete $opt{help} ) {
require Pod::Usage;
commit 52c84c1ce5c939250f8114653e0cfc0e3ae0287d
Author: Jim Brandt <jbrandt at bestpractical.com>
Date: Wed Feb 14 15:46:22 2024 -0500
Add an option to set MaxResults
diff --git a/bin/rt-import-aws-assets.in b/bin/rt-import-aws-assets.in
index 0537a74..6f2dbcd 100644
--- a/bin/rt-import-aws-assets.in
+++ b/bin/rt-import-aws-assets.in
@@ -33,7 +33,8 @@ sub run{
Region => $args{'region'},
ServiceType => $args{'type'},
ReservedInstances => $args{'reserved'},
- Token => $token );
+ Token => $token,
+ MaxResults => $args{'results'} );
RT::Extension::AWS::Assets::InsertAWSAssets(
AWSResources => $aws_resources,
@@ -54,7 +55,8 @@ sub run{
Region => $args{'region'},
ServiceType => $args{'type'},
ReservedInstances => $args{'reserved'},
- Token => $token );
+ Token => $token,
+ MaxResults => $args{'results'} );
RT::Extension::AWS::Assets::UpdateAWSAssets(
AWSResources => $aws_resources,
@@ -74,7 +76,7 @@ sub process_args {
local @ARGV = @_;
my %opt;
- Getopt::Long::GetOptions( \%opt, 'help|h', 'insert|i', 'update|u', 'catalog=s', 'type=s', 'reserved', 'region=s', 'debug|d' );
+ Getopt::Long::GetOptions( \%opt, 'help|h', 'insert|i', 'update|u', 'catalog=s', 'type=s', 'reserved', 'region=s', 'results=i', 'debug|d' );
if ( delete $opt{help} ) {
require Pod::Usage;
@@ -87,6 +89,10 @@ sub process_args {
exit;
}
+ unless ( $opt{'results'} ) {
+ $opt{'results'} = 20;
+ }
+
return %opt;
}
diff --git a/lib/RT/Extension/AWS/Assets.pm b/lib/RT/Extension/AWS/Assets.pm
index 0777ffb..ca52320 100644
--- a/lib/RT/Extension/AWS/Assets.pm
+++ b/lib/RT/Extension/AWS/Assets.pm
@@ -153,7 +153,7 @@ sub FetchSingleAssetFromAWS {
sub FetchMultipleAssetsFromAWS {
my %args = (
- MaxResults => 20,
+ MaxResults => 100,
Token => undef,
@_,
);
commit 68f7dfb8d1406bf93119772832649675463821d9
Author: Jim Brandt <jbrandt at bestpractical.com>
Date: Wed Feb 14 15:35:05 2024 -0500
Show only assets with no existing links
diff --git a/html/AWS/LinkAsset.html b/html/AWS/LinkAsset.html
index 11785a4..fe83668 100644
--- a/html/AWS/LinkAsset.html
+++ b/html/AWS/LinkAsset.html
@@ -68,6 +68,8 @@ if ( $AssetObj->FirstCustomFieldValue('Service Type') eq 'RDS' ) {
$Query .= " AND 'CF.{Engine}' = '$engine'";
}
+$Query .= " AND DependsOn IS NULL";
+
my $assets = RT::Assets->new( $session{'CurrentUser'} );
my ($ok, $msg) = $assets->FromSQL( $Query );
commit 98a69c5c092821751e21f41eccbb8e78f59a52cd
Author: Jim Brandt <jbrandt at bestpractical.com>
Date: Wed Feb 14 15:31:18 2024 -0500
Convert postgresql to postgres for AWS RDS search
RDS reservations use postgresql and RDS instances use postgres,
so convert when generating the lookup for unlinked RDS instances.
diff --git a/html/AWS/LinkAsset.html b/html/AWS/LinkAsset.html
index 666f1cc..11785a4 100644
--- a/html/AWS/LinkAsset.html
+++ b/html/AWS/LinkAsset.html
@@ -60,7 +60,12 @@ $Query = "Catalog = '" . RT->Config->Get('AWSAssetsInstanceCatalog')
. "' AND 'CF.{Instance Type}' = '" . $AssetObj->FirstCustomFieldValue('Instance Type') . "'";
if ( $AssetObj->FirstCustomFieldValue('Service Type') eq 'RDS' ) {
- $Query .= " AND 'CF.{Engine}' = '" . $AssetObj->FirstCustomFieldValue('Product Description') . "'";
+ my $engine = $AssetObj->FirstCustomFieldValue('Product Description');
+
+ # Fix inconsistent postgresql vs. postgres
+ $engine = 'postgres' if $engine eq 'postgresql';
+
+ $Query .= " AND 'CF.{Engine}' = '$engine'";
}
my $assets = RT::Assets->new( $session{'CurrentUser'} );
commit 7c2801533ae32e791b1a456e54c2fe955522531a
Author: Jim Brandt <jbrandt at bestpractical.com>
Date: Tue Feb 13 13:45:57 2024 -0500
Initial version
diff --git a/Changes b/Changes
index d04f544..a63b939 100644
--- a/Changes
+++ b/Changes
@@ -1,4 +1,4 @@
Revision history for RT-Extension-AWS-Assets
-0.01 [Release Date]
+0.01 2024-02-13
- Initial version
diff --git a/MANIFEST b/MANIFEST
new file mode 100644
index 0000000..4d54687
--- /dev/null
+++ b/MANIFEST
@@ -0,0 +1,24 @@
+bin/rt-import-aws-assets.in
+Changes
+html/AWS/LinkAsset.html
+html/Callbacks/RT-Extension-AWS-Assets/Asset/Display.html/BeforeProcessArguments
+html/Callbacks/RT-Extension-AWS-Assets/Elements/Tabs/Privileged
+inc/Module/Install.pm
+inc/Module/Install/Base.pm
+inc/Module/Install/Can.pm
+inc/Module/Install/Fetch.pm
+inc/Module/Install/Include.pm
+inc/Module/Install/Makefile.pm
+inc/Module/Install/Metadata.pm
+inc/Module/Install/ReadmeFromPod.pm
+inc/Module/Install/RTx.pm
+inc/Module/Install/RTx/Runtime.pm
+inc/Module/Install/Substitute.pm
+inc/Module/Install/Win32.pm
+inc/Module/Install/WriteAll.pm
+inc/YAML/Tiny.pm
+lib/RT/Extension/AWS/Assets.pm
+Makefile.PL
+MANIFEST This list of files
+META.yml
+README
diff --git a/META.yml b/META.yml
index bcc2277..143ceb3 100644
--- a/META.yml
+++ b/META.yml
@@ -16,8 +16,10 @@ meta-spec:
name: RT-Extension-AWS-Assets
no_index:
directory:
+ - html
- inc
requires:
+ Paws: 0
perl: 5.10.1
resources:
license: http://opensource.org/licenses/gpl-license.php
diff --git a/README b/README
index 8d40d91..7843050 100644
--- a/README
+++ b/README
@@ -23,6 +23,9 @@ INSTALLATION
Restart your webserver
+METHODS
+ Accept a loaded RT::Asset object and a Paws Instance object.
+
AUTHOR
All bugs should be reported via email to
bug-RT-Extension-AWS-Assets at rt.cpan.org
commit f93a5df87a629cf5e31ffa1798bd9e1f4276e2a2
Author: Jim Brandt <jbrandt at bestpractical.com>
Date: Tue Feb 13 11:42:08 2024 -0500
Add reservation linking page
diff --git a/html/AWS/LinkAsset.html b/html/AWS/LinkAsset.html
new file mode 100644
index 0000000..666f1cc
--- /dev/null
+++ b/html/AWS/LinkAsset.html
@@ -0,0 +1,88 @@
+<& /Elements/Header, Title => loc("Link Reservation #[_1]: [_2], Instance Type: [_3]", $AssetObj->id, $AssetObj->Name, $AssetObj->FirstCustomFieldValue('Instance Type')) &>
+<& /Elements/Tabs &>
+
+% $m->callback(CallbackName => 'BeforeActionList', ARGSRef => \%ARGS, Asset => $AssetObj, Results => \@results);
+
+<& /Elements/ListActions, actions => \@results &>
+
+<form action="<% RT->Config->Get('WebPath') %>/AWS/LinkAsset.html" method="post">
+<input type="hidden" name="id" value="<% $id %>" />
+<div><p class="text-center">Running with query: <% $Query %></p></div>
+
+<& /Elements/CollectionList,
+ %ARGS,
+ Collection => $assets,
+ Query => $Query,
+ OrderBy => $OrderBy,
+ Order => $Order,
+ Rows => $Rows,
+ DisplayFormat => "'__RadioButton.{SelectedAsset}__', $Format",
+ Format => $Format,
+ Class => 'RT::Assets',
+ ObjectType => 'RT::Asset',
+ AllowSorting => 0,
+ ShowNavigation => 0,
+ InlineEdit => 0,
+&>
+
+<& /Elements/Submit,
+ Name => "LinkReservation",
+ Caption => "Link Reservation",
+ Label => loc("Link"),
+&>
+</form>
+
+<%init>
+
+my @results;
+my $AssetObj = LoadAsset($id);
+
+$m->callback(CallbackName => 'Initial', %ARGS, AssetObj => $AssetObj);
+
+if ( $ARGS{'SelectedAsset'} ) {
+ # Add a DependedOnBy from the id on the page to the selected asset
+ my $link_asset = LoadAsset($ARGS{'SelectedAsset'});
+ my ($ok, $msg) = $link_asset->AddLink( Target => 'asset:' . $AssetObj->Id, Type => 'DependsOn' );
+
+ push @results, $msg;
+ if ( not $ok ) {
+ RT->Logger->error('Unable to add link to asset ' . $ARGS{'SelectedAsset'} . ": $msg");
+ }
+}
+
+# By default, we want to find assets in the $AWSAssetsInstanceCatalog that match
+# for EC2: Region, Service Type, Instance Type
+# for RDS: Region, Service Type, Instance Type, Engine
+
+$Query = "Catalog = '" . RT->Config->Get('AWSAssetsInstanceCatalog')
+ . "' AND 'CF.{Region}' = '" . $AssetObj->FirstCustomFieldValue('Region')
+ . "' AND 'CF.{Service Type}' = '" . $AssetObj->FirstCustomFieldValue('Service Type')
+ . "' AND 'CF.{Instance Type}' = '" . $AssetObj->FirstCustomFieldValue('Instance Type') . "'";
+
+if ( $AssetObj->FirstCustomFieldValue('Service Type') eq 'RDS' ) {
+ $Query .= " AND 'CF.{Engine}' = '" . $AssetObj->FirstCustomFieldValue('Product Description') . "'";
+}
+
+my $assets = RT::Assets->new( $session{'CurrentUser'} );
+my ($ok, $msg) = $assets->FromSQL( $Query );
+
+unless ( $ok ) {
+ RT->Logger->error("Error loading assets: $msg");
+}
+
+MaybeRedirectForResults(
+ Actions => \@results,
+ Path => '/Asset/Display.html',
+ Arguments => { id => $AssetObj->Id },
+);
+
+</%init>
+<%args>
+$id => undef
+
+$Query => undef
+$Format => RT->Config->Get('AWSAssetsLinkFormat')
+$Rows => 20
+$OrderBy => RT->Config->Get('DefaultSearchResultOrderBy')
+$Order => RT->Config->Get('DefaultSearchResultOrder')
+</%args>
commit b7ae20b5fecb3903c48d604b5282122c128519ab
Author: Jim Brandt <jbrandt at bestpractical.com>
Date: Mon Feb 12 15:00:56 2024 -0500
Handle multiple reservations
diff --git a/html/Callbacks/RT-Extension-AWS-Assets/Asset/Display.html/BeforeProcessArguments b/html/Callbacks/RT-Extension-AWS-Assets/Asset/Display.html/BeforeProcessArguments
index 2119617..6cded4a 100644
--- a/html/Callbacks/RT-Extension-AWS-Assets/Asset/Display.html/BeforeProcessArguments
+++ b/html/Callbacks/RT-Extension-AWS-Assets/Asset/Display.html/BeforeProcessArguments
@@ -4,14 +4,26 @@ my ($result, $message);
if ( $ARGSRef->{'AWSUpdate'} && $ARGSRef->{'AWSUpdate'} eq 'true' ) {
+ my $reserved = 0;
+ $reserved = 1 if $AssetObj->CatalogObj->Name eq RT->Config->Get('AWSAssetsReservedInstancesCatalog');
+
+ my $asset_id_cf = 'AWS ID';
+ $asset_id_cf = 'AWS Reserved Instance ID' if $reserved;
+
# Make sure we have required params
- if ( $AssetObj->FirstCustomFieldValue('AWS ID')
+ if ( $AssetObj->FirstCustomFieldValue("$asset_id_cf")
&& $AssetObj->FirstCustomFieldValue('Service Type')
&& $AssetObj->FirstCustomFieldValue('Region') ) {
- RT::Extension::AWS::Assets::ReloadFromAWS($AssetObj);
- $result = 1;
- $message = 'Asset information updated from AWS.';
+ $result = RT::Extension::AWS::Assets::ReloadFromAWS($AssetObj, $asset_id_cf, $reserved);
+
+ if ( $result ) {
+ $message = 'Asset information updated from AWS.';
+ }
+ else {
+ $message = "Error retrieving information from AWS, ask your RT admin for details.";
+ }
+
}
else {
$result = 0;
diff --git a/html/Callbacks/RT-Extension-AWS-Assets/Elements/Tabs/Privileged b/html/Callbacks/RT-Extension-AWS-Assets/Elements/Tabs/Privileged
index edfff70..4254d4a 100644
--- a/html/Callbacks/RT-Extension-AWS-Assets/Elements/Tabs/Privileged
+++ b/html/Callbacks/RT-Extension-AWS-Assets/Elements/Tabs/Privileged
@@ -8,6 +8,11 @@ return unless $id;
my $asset = LoadAsset($id);
return unless $asset->Id;
+# Only for the AWS catalogs
+my $catalog_name = $asset->CatalogObj->Name;
+return unless $catalog_name eq RT->Config->Get('AWSAssetsInstanceCatalog')
+ || $catalog_name eq RT->Config->Get('AWSAssetsReservedInstancesCatalog');
+
if ( $asset->CurrentUserHasRight('ModifyAsset') ){
PageMenu()->child( 'actions' )->child(
'update_from_aws' => title => loc( 'Update from AWS' ),
diff --git a/lib/RT/Extension/AWS/Assets.pm b/lib/RT/Extension/AWS/Assets.pm
index 07ddce9..0777ffb 100644
--- a/lib/RT/Extension/AWS/Assets.pm
+++ b/lib/RT/Extension/AWS/Assets.pm
@@ -53,15 +53,23 @@ Add this line:
sub ReloadFromAWS {
my $asset = shift;
+ my $asset_id_cf = shift;
+ my $reserved = shift;
my $res_obj = FetchSingleAssetFromAWS(
- AWSID => $asset->FirstCustomFieldValue('AWS ID'),
+ AWSID => $asset->FirstCustomFieldValue("$asset_id_cf"),
ServiceType => $asset->FirstCustomFieldValue('Service Type'),
- Region => $asset->FirstCustomFieldValue('Region'));
+ Region => $asset->FirstCustomFieldValue('Region'),
+ ReservedInstances => $reserved );
- UpdateAWSAsset($asset, $res_obj);
-}
+ return unless $res_obj;
+
+ UpdateAWSAsset( AssetObj => $asset, PawsObj => $res_obj,
+ Service => $asset->FirstCustomFieldValue('Service Type'),
+ ReservedInstances => $reserved );
+ return 1;
+}
sub AWSCredentials {
my $credentials = Paws::Credential::Explicit->new(
@@ -74,6 +82,9 @@ sub AWSCredentials {
sub FetchSingleAssetFromAWS {
my %args = @_;
+ # Filtering not working now, will fix later
+ return;
+
unless ( $args{'AWSID'} ) {
RT->Logger->error('RT-Extension-AWS-Assets: No AWS ID found.');
return;
@@ -90,11 +101,46 @@ sub FetchSingleAssetFromAWS {
my $method = 'DescribeInstances'; # Default for EC2
$method = 'DescribeDBInstances' if $args{'ServiceType'} eq 'RDS';
- eval {
- my $service = Paws->service($args{'ServiceType'}, credentials => $credentials, region => $args{'Region'});
- my $res = $service->$method(InstanceIds => [$args{'AWSID'}]);
- $instance_obj = $res->Reservations->[0]->Instances->[0];
- };
+
+ if ( $args{'ServiceType'} eq 'EC2' ) {
+ if ( $args{'ReservedInstances'} ) {
+ eval {
+ my $service = Paws->service($args{'ServiceType'}, credentials => $credentials, region => $args{'Region'});
+
+ # No paging for reserved instance API
+ my $res = $service->DescribeReservedInstances(ReservedInstancesIds => [$args{'AWSID'}]);
+ $instance_obj = $res->Reservations->[0]->Instances->[0];
+ };
+ }
+ else {
+ eval {
+ my $service = Paws->service($args{'ServiceType'}, credentials => $credentials, region => $args{'Region'});
+ my $res = $service->$method(InstanceIds => [$args{'AWSID'}]);
+ $instance_obj = $res->Reservations->[0]->Instances->[0];
+ };
+ }
+ }
+ elsif ( $args{'ServiceType'} eq 'RDS' ) {
+ if ( $args{'ReservedInstances'} ) {
+ eval {
+ my $service = Paws->service($args{'ServiceType'}, credentials => $credentials, region => $args{'Region'});
+
+ # The RDS version does have paging, but leaving it out for consistency with EC2
+ # Set Max to the Max allowed. Will need to update when we go over 100 in a region
+ my $res = $service->DescribeReservedDBInstances(ReservedDBInstanceId => $args{'AWSID'});
+ $instance_obj = $res->ReservedDBInstances;
+ };
+ }
+ else {
+ eval {
+ my $service = Paws->service($args{'ServiceType'}, credentials => $credentials, region => $args{'Region'});
+ # Doesn't work now, not sure why, Values are passed as null
+# my $res = $service->DescribeDBInstances(Filters => [{ Name => 'dbi-resource-id', Values => ["foo", "bar"] }]);
+ my $res = $service->DescribeDBInstances(DBInstanceIdentifier => );
+ $instance_obj = $res->DBInstances;
+ };
+ }
+ }
if ( $@ ) {
RT->Logger->error("RT-Extension-AWS-Assets: Failed call to AWS: " . $@);
@@ -303,11 +349,13 @@ sub InsertAWSAssets {
for my $resource ( @{ $args{'AWSResources'} } ) {
my $resource_id;
my $instance;
-warn "got is: " . p($resource);
+ my $count = 1; # Regular EC2 and RDS are always 1
+
if ( $args{'ServiceType'} eq 'EC2' ) {
if ( $args{'ReservedInstances'} ) {
$instance = $resource;
$resource_id = $resource->ReservedInstancesId;
+ $count = $instance->InstanceCount;
}
else {
$instance = $resource->Instances->[0];
@@ -318,6 +366,7 @@ warn "got is: " . p($resource);
if ( $args{'ReservedInstances'} ) {
$instance = $resource;
$resource_id = $resource->LeaseId;
+ $count = $instance->DBInstanceCount;
}
else {
$instance = $resource;
@@ -331,24 +380,23 @@ warn "got is: " . p($resource);
my ($ok, $msg) = $assets->FromSQL("Catalog = '" . $catalog .
"' AND 'CF.{$asset_id_cf}' = '" . $resource_id . "'");
- # AWS ID is unique, so there should only ever be 1 or 0
- my $asset = $assets->First;
+ my $asset_exists = 0;
+ $asset_exists = 1 if $assets->Count >= $count;
- # Search for an existing asset, next if found
# Asset already exists, next
RT->Logger->debug("Asset for " . $resource_id . " exists, skipping")
- if $asset and $asset->Id;
- next if $asset and $asset->Id;
+ if $asset_exists;
+ next if $asset_exists;
- # Try to create a new asset with AWS ID, Region, Service Type
- my $new_asset = RT::Asset->new($args{'CurrentUser'});
+ # Create an empty asset to more easily load the CF ids
+ my $void_asset = RT::Asset->new($args{'CurrentUser'});
my $aws_id_cf;
if ( $args{'ReservedInstances'} ) {
- $aws_id_cf = LoadCustomFieldByIdentifier($new_asset, 'AWS Reserved Instance ID', $catalog, $args{'CurrentUser'});
+ $aws_id_cf = LoadCustomFieldByIdentifier($void_asset, 'AWS Reserved Instance ID', $catalog, $args{'CurrentUser'});
}
else {
- $aws_id_cf = LoadCustomFieldByIdentifier($new_asset, 'AWS ID', $catalog, $args{'CurrentUser'});
+ $aws_id_cf = LoadCustomFieldByIdentifier($void_asset, 'AWS ID', $catalog, $args{'CurrentUser'});
}
unless ( $aws_id_cf && $aws_id_cf->Id ) {
@@ -356,36 +404,43 @@ warn "got is: " . p($resource);
next;
}
- my $region_cf = LoadCustomFieldByIdentifier($new_asset, 'Region', $catalog, $args{'CurrentUser'});
+ my $region_cf = LoadCustomFieldByIdentifier($void_asset, 'Region', $catalog, $args{'CurrentUser'});
unless ( $region_cf && $region_cf->Id ) {
RT->Logger->error('Unable to load Region CF for asset');
next;
}
- my $service_type_cf = LoadCustomFieldByIdentifier($new_asset, 'Service Type', $catalog, $args{'CurrentUser'});
+ my $service_type_cf = LoadCustomFieldByIdentifier($void_asset, 'Service Type', $catalog, $args{'CurrentUser'});
unless ( $service_type_cf && $service_type_cf->Id ) {
RT->Logger->error('Unable to load Service Type CF for asset');
next;
}
- ($ok, $msg) = $new_asset->Create(
- Catalog => $catalog,
- 'CustomField-' . $aws_id_cf->Id => $resource_id,
- 'CustomField-' . $region_cf->Id => $args{'Region'},
- 'CustomField-' . $service_type_cf->Id => $args{'ServiceType'},
- );
+ my $created = 0;
+ while ( $created < $count ) {
+ # Try to create a new asset with AWS ID, Region, Service Type
+
+ my $new_asset = RT::Asset->new($args{'CurrentUser'});
+ ($ok, $msg) = $new_asset->Create(
+ Catalog => $catalog,
+ 'CustomField-' . $aws_id_cf->Id => $resource_id,
+ 'CustomField-' . $region_cf->Id => $args{'Region'},
+ 'CustomField-' . $service_type_cf->Id => $args{'ServiceType'},
+ );
+ $created++;
+
+ if ( not $ok ) {
+ RT->Logger->error('Unable to create new asset for instance ' . $resource_id . ' ' . $msg);
+ next;
+ }
+ else {
+ RT->Logger->debug('Created asset ' . $new_asset->Id . ' for instance ' . $resource_id);
+ }
- if ( not $ok ) {
- RT->Logger->error('Unable to create new asset for instance ' . $resource_id . ' ' . $msg);
- next;
+ # Call UpdateAWSAsset to load remaining CFs
+ UpdateAWSAsset( AssetObj => $new_asset, PawsObj => $instance,
+ Service => $args{'ServiceType'}, ReservedInstances => $args{'ReservedInstances'});
}
- else {
- RT->Logger->debug('Created asset ' . $new_asset->Id . ' for instance ' . $resource_id);
- }
-
- # Call UpdateAWSAsset to load remaining CFs
- UpdateAWSAsset( AssetObj => $new_asset, PawsObj => $instance,
- Service => $args{'ServiceType'}, ReservedInstances => $args{'ReservedInstances'});
}
return;
}
@@ -398,36 +453,57 @@ sub UpdateAWSAssets {
my $catalog = RT->Config->Get('AWSAssetsInstanceCatalog');
$catalog = RT->Config->Get('AWSAssetsReservedInstancesCatalog') if $args{'ReservedInstances'};
+ my $asset_id_cf = 'AWS ID';
+ $asset_id_cf = 'AWS Reserved Instance ID' if $args{'ReservedInstances'};
+
for my $resource ( @{ $args{'AWSResources'} } ) {
my $resource_id;
my $instance;
+ my $count = 1; # Regular EC2 and RDS are always 1
if ( $args{'ServiceType'} eq 'EC2' ) {
- $instance = $resource->Instances->[0];
- $resource_id = $resource->Instances->[0]->InstanceId;
+ if ( $args{'ReservedInstances'} ) {
+ $instance = $resource;
+ $resource_id = $resource->ReservedInstancesId;
+ $count = $instance->InstanceCount;
+ }
+ else {
+ $instance = $resource->Instances->[0];
+ $resource_id = $resource->Instances->[0]->InstanceId;
+ }
}
elsif ( $args{'ServiceType'} eq 'RDS' ) {
- $instance = $resource;
- $resource_id = $resource->DbiResourceId;
+ if ( $args{'ReservedInstances'} ) {
+ $instance = $resource;
+ $resource_id = $resource->LeaseId;
+ $count = $instance->DBInstanceCount;
+ }
+ else {
+ $instance = $resource;
+ $resource_id = $resource->DbiResourceId;
+ }
}
# Load as system user to find all possible assets to avoid
# trying to create a duplicate CurrentUser might not be able to see
my $assets = RT::Assets->new( RT->SystemUser );
my ($ok, $msg) = $assets->FromSQL("Catalog = '" . $catalog .
- "' AND 'CF.{AWS ID}' = '" . $resource_id . "'");
+ "' AND 'CF.{$asset_id_cf}' = '" . $resource_id . "'");
- # AWS ID is unique, so there should only ever be 1 or 0
- my $asset = $assets->First;
+ my $asset_found = 0;
+ $asset_found = 1 if $assets->Count;
- unless ( $asset and $asset->Id ) {
+ unless ( $asset_found ) {
RT->Logger->debug("No asset found for " . $resource_id . ", skipping");
next;
}
- UpdateAWSAsset( AssetObj => $asset, PawsObj => $instance,
- Service => $args{'ServiceType'}, ReservedInstances => $args{'ReservedInstances'});
- RT->Logger->debug('Updated asset ' . $asset->Id . ' ' . $asset->Name);
+ $assets->RedoSearch;
+ while ( my $asset = $assets->Next ) {
+ UpdateAWSAsset( AssetObj => $asset, PawsObj => $instance,
+ Service => $args{'ServiceType'}, ReservedInstances => $args{'ReservedInstances'});
+ RT->Logger->debug('Updated asset ' . $asset->Id . ' ' . $asset->Name);
+ }
}
return;
}
commit e9a4aef303e4008fb6e6fd0b691599f28ad6a636
Author: Jim Brandt <jbrandt at bestpractical.com>
Date: Mon Feb 12 10:47:46 2024 -0500
Sync EC2 and RDS reserved instances to assets
diff --git a/bin/rt-import-aws-assets.in b/bin/rt-import-aws-assets.in
index ad285df..0537a74 100644
--- a/bin/rt-import-aws-assets.in
+++ b/bin/rt-import-aws-assets.in
@@ -32,12 +32,14 @@ sub run{
RT::Extension::AWS::Assets::FetchMultipleAssetsFromAWS(
Region => $args{'region'},
ServiceType => $args{'type'},
+ ReservedInstances => $args{'reserved'},
Token => $token );
RT::Extension::AWS::Assets::InsertAWSAssets(
AWSResources => $aws_resources,
Region => $args{'region'},
ServiceType => $args{'type'},
+ ReservedInstances => $args{'reserved'},
CurrentUser => GetCurrentUser() );
} while ( $token )
@@ -51,12 +53,14 @@ sub run{
RT::Extension::AWS::Assets::FetchMultipleAssetsFromAWS(
Region => $args{'region'},
ServiceType => $args{'type'},
+ ReservedInstances => $args{'reserved'},
Token => $token );
RT::Extension::AWS::Assets::UpdateAWSAssets(
AWSResources => $aws_resources,
Region => $args{'region'},
ServiceType => $args{'type'},
+ ReservedInstances => $args{'reserved'},
CurrentUser => GetCurrentUser() );
} while ( $token )
@@ -70,7 +74,7 @@ sub process_args {
local @ARGV = @_;
my %opt;
- Getopt::Long::GetOptions( \%opt, 'help|h', 'insert|i', 'update|u', 'catalog=s', 'type=s', 'region=s', 'debug|d' );
+ Getopt::Long::GetOptions( \%opt, 'help|h', 'insert|i', 'update|u', 'catalog=s', 'type=s', 'reserved', 'region=s', 'debug|d' );
if ( delete $opt{help} ) {
require Pod::Usage;
diff --git a/lib/RT/Extension/AWS/Assets.pm b/lib/RT/Extension/AWS/Assets.pm
index 8d144ce..07ddce9 100644
--- a/lib/RT/Extension/AWS/Assets.pm
+++ b/lib/RT/Extension/AWS/Assets.pm
@@ -128,34 +128,57 @@ sub FetchMultipleAssetsFromAWS {
my $token;
if ( $args{'ServiceType'} eq 'EC2' ) {
- eval {
- my $service = Paws->service($args{'ServiceType'}, credentials => $credentials, region => $args{'Region'});
+ if ( $args{'ReservedInstances'} ) {
+ eval {
+ my $service = Paws->service($args{'ServiceType'}, credentials => $credentials, region => $args{'Region'});
+
+ # No paging for reserved instance API
+ $res = $service->DescribeReservedInstances();
+ $aws_resources = $res->ReservedInstances;
+ };
+ }
+ else {
+ eval {
+ my $service = Paws->service($args{'ServiceType'}, credentials => $credentials, region => $args{'Region'});
- if ( $args{'Token'} ) {
- $res = $service->DescribeInstances(MaxResults => $args{'MaxResults'}, NextToken => $args{'Token'});
- }
- else {
- $res = $service->DescribeInstances(MaxResults => $args{'MaxResults'});
- }
+ if ( $args{'Token'} ) {
+ $res = $service->DescribeInstances(MaxResults => $args{'MaxResults'}, NextToken => $args{'Token'});
+ }
+ else {
+ $res = $service->DescribeInstances(MaxResults => $args{'MaxResults'});
+ }
- $aws_resources = $res->Reservations;
- $token = $res->NextToken;
- };
+ $aws_resources = $res->Reservations;
+ $token = $res->NextToken;
+ };
+ }
}
elsif ( $args{'ServiceType'} eq 'RDS' ) {
- eval {
- my $service = Paws->service($args{'ServiceType'}, credentials => $credentials, region => $args{'Region'});
+ if ( $args{'ReservedInstances'} ) {
+ eval {
+ my $service = Paws->service($args{'ServiceType'}, credentials => $credentials, region => $args{'Region'});
+
+ # The RDS version does have paging, but leaving it out for consistency with EC2
+ # Set Max to the Max allowed. Will need to update when we go over 100 in a region
+ $res = $service->DescribeReservedDBInstances(MaxRecords => 100);
+ $aws_resources = $res->ReservedDBInstances;
+ };
+ }
+ else {
+ eval {
+ my $service = Paws->service($args{'ServiceType'}, credentials => $credentials, region => $args{'Region'});
- if ( $args{'Marker'} ) {
- $res = $service->DescribeDBInstances(MaxRecords => $args{'MaxResults'}, Marker => $args{'Token'});
- }
- else {
- $res = $service->DescribeDBInstances(MaxRecords => $args{'MaxResults'});
- }
+ if ( $args{'Marker'} ) {
+ $res = $service->DescribeDBInstances(MaxRecords => $args{'MaxResults'}, Marker => $args{'Token'});
+ }
+ else {
+ $res = $service->DescribeDBInstances(MaxRecords => $args{'MaxResults'});
+ }
- $aws_resources = $res->DBInstances;
- $token = $res->Marker;
- };
+ $aws_resources = $res->DBInstances;
+ $token = $res->Marker;
+ };
+ }
}
if ( $@ ) {
@@ -172,11 +195,18 @@ Accept a loaded RT::Asset object and a Paws Instance object.
=cut
sub UpdateAWSAsset {
- my $asset = shift;
- my $paws_obj = shift;
- my $service = shift;
+ my %args = @_;
+ my $asset = $args{'AssetObj'};
+ my $paws_obj = $args{'PawsObj'};
+ my $service = $args{'Service'};
+ my $reserved = $args{'ReservedInstances'};
+
+ # For looking up the field mapping, use RI (Reserved Instances) for the service
+ # rather than the AWS service the RI is for.
+ my $config_key = $service;
+ $config_key .= ':RI' if $reserved;
- foreach my $aws_value ( @{ RT->Config->Get('AWSAssetsUpdateFields')->{$service} } ) {
+ foreach my $aws_value ( @{ RT->Config->Get('AWSAssetsUpdateFields')->{$config_key} } ) {
my $submethod;
my $cf_name = $aws_value;
@@ -187,12 +217,34 @@ sub UpdateAWSAsset {
# Fixup some special cases
if ( $service eq 'RDS' ) {
- $method = 'DBInstanceIdentifier' if $cf_name eq 'Name';
- $method = 'DBInstanceClass' if $cf_name eq 'Instance Type';
+ $method = 'DBInstanceIdentifier' if ( $cf_name eq 'Name' );
+ $method = 'DBInstanceClass' if ( $cf_name eq 'Instance Type' );
+ }
+
+ if ( $reserved ) {
+ $method = 'ProductDescription' if ( $cf_name eq 'Platform' );
+ $method = 'InstanceTenancy' if ( $cf_name eq 'Tenancy' );
+ $method = 'Start' if ( $service eq 'EC2' && $cf_name eq 'Reservation Start' );
+ $method = 'StartTime' if ( $service eq 'RDS' && $cf_name eq 'Reservation Start' );
+ $method = 'End' if ( $cf_name eq 'Reservation End' );
+ $method = 'InstanceType' if ( $service eq 'EC2' && $cf_name eq 'Name' );
+ $method = 'ReservedDBInstanceId' if ( $service eq 'RDS' && $cf_name eq 'Name' );
}
+ # Fixups (mostly) done, start setting values
my ($ret, $msg);
+ # RDS RIs don't provide EndTime as a value via the API, even though EC2 does
+ # Calculate it here using Start and Duration.
+ if ( $reserved && $service eq 'RDS' && $cf_name eq 'Reservation End' ) {
+ my $end = RT::Date->new(RT->SystemUser);
+ $end->Set( Format => 'unknown', Value => $paws_obj->StartTime );
+ $end->AddSeconds($paws_obj->Duration);
+
+ ($ret, $msg) = $asset->AddCustomFieldValue( Field => $cf_name, Value => $end->ISO );
+ next;
+ }
+
if ( $submethod && ( $submethod eq 'Tags' || $submethod eq 'TagList' ) ) {
foreach my $tag ( @{ $paws_obj->$submethod } ) {
if ( $tag->Key eq $cf_name ) {
@@ -208,7 +260,7 @@ sub UpdateAWSAsset {
}
}
}
- elsif ( $method eq 'DBInstanceIdentifier' ) {
+ elsif ( $cf_name eq 'Name' ) {
# Name isn't a CF but a core asset field
($ret, $msg) = $asset->SetName($paws_obj->$method);
}
@@ -243,25 +295,41 @@ sub InsertAWSAssets {
);
my $catalog = RT->Config->Get('AWSAssetsInstanceCatalog');
+ $catalog = RT->Config->Get('AWSAssetsReservedInstancesCatalog') if $args{'ReservedInstances'};
+
+ my $asset_id_cf = 'AWS ID';
+ $asset_id_cf = 'AWS Reserved Instance ID' if $args{'ReservedInstances'};
for my $resource ( @{ $args{'AWSResources'} } ) {
my $resource_id;
my $instance;
-
+warn "got is: " . p($resource);
if ( $args{'ServiceType'} eq 'EC2' ) {
- $instance = $resource->Instances->[0];
- $resource_id = $resource->Instances->[0]->InstanceId;
+ if ( $args{'ReservedInstances'} ) {
+ $instance = $resource;
+ $resource_id = $resource->ReservedInstancesId;
+ }
+ else {
+ $instance = $resource->Instances->[0];
+ $resource_id = $resource->Instances->[0]->InstanceId;
+ }
}
elsif ( $args{'ServiceType'} eq 'RDS' ) {
- $instance = $resource;
- $resource_id = $resource->DbiResourceId;
+ if ( $args{'ReservedInstances'} ) {
+ $instance = $resource;
+ $resource_id = $resource->LeaseId;
+ }
+ else {
+ $instance = $resource;
+ $resource_id = $resource->DbiResourceId;
+ }
}
# Load as system user to find all possible assets to avoid
# trying to create a duplicate CurrentUser might not be able to see
my $assets = RT::Assets->new( RT->SystemUser );
my ($ok, $msg) = $assets->FromSQL("Catalog = '" . $catalog .
- "' AND 'CF.{AWS ID}' = '" . $resource_id . "'");
+ "' AND 'CF.{$asset_id_cf}' = '" . $resource_id . "'");
# AWS ID is unique, so there should only ever be 1 or 0
my $asset = $assets->First;
@@ -275,7 +343,14 @@ sub InsertAWSAssets {
# Try to create a new asset with AWS ID, Region, Service Type
my $new_asset = RT::Asset->new($args{'CurrentUser'});
- my $aws_id_cf = LoadCustomFieldByIdentifier($new_asset, 'AWS ID', $catalog, $args{'CurrentUser'});
+ my $aws_id_cf;
+ if ( $args{'ReservedInstances'} ) {
+ $aws_id_cf = LoadCustomFieldByIdentifier($new_asset, 'AWS Reserved Instance ID', $catalog, $args{'CurrentUser'});
+ }
+ else {
+ $aws_id_cf = LoadCustomFieldByIdentifier($new_asset, 'AWS ID', $catalog, $args{'CurrentUser'});
+ }
+
unless ( $aws_id_cf && $aws_id_cf->Id ) {
RT->Logger->error('Unable to load AWS ID CF for asset');
next;
@@ -294,7 +369,7 @@ sub InsertAWSAssets {
}
($ok, $msg) = $new_asset->Create(
- Catalog => RT->Config->Get('AWSAssetsInstanceCatalog'),
+ Catalog => $catalog,
'CustomField-' . $aws_id_cf->Id => $resource_id,
'CustomField-' . $region_cf->Id => $args{'Region'},
'CustomField-' . $service_type_cf->Id => $args{'ServiceType'},
@@ -309,7 +384,8 @@ sub InsertAWSAssets {
}
# Call UpdateAWSAsset to load remaining CFs
- UpdateAWSAsset($new_asset, $instance, $args{'ServiceType'});
+ UpdateAWSAsset( AssetObj => $new_asset, PawsObj => $instance,
+ Service => $args{'ServiceType'}, ReservedInstances => $args{'ReservedInstances'});
}
return;
}
@@ -320,6 +396,7 @@ sub UpdateAWSAssets {
);
my $catalog = RT->Config->Get('AWSAssetsInstanceCatalog');
+ $catalog = RT->Config->Get('AWSAssetsReservedInstancesCatalog') if $args{'ReservedInstances'};
for my $resource ( @{ $args{'AWSResources'} } ) {
my $resource_id;
@@ -348,7 +425,8 @@ sub UpdateAWSAssets {
next;
}
- UpdateAWSAsset($asset, $instance, $args{'ServiceType'});
+ UpdateAWSAsset( AssetObj => $asset, PawsObj => $instance,
+ Service => $args{'ServiceType'}, ReservedInstances => $args{'ReservedInstances'});
RT->Logger->debug('Updated asset ' . $asset->Id . ' ' . $asset->Name);
}
return;
-----------------------------------------------------------------------
Summary of changes:
Changes | 2 +-
MANIFEST | 24 ++
META.yml | 2 +
README | 3 +
bin/rt-import-aws-assets.in | 20 +-
html/AWS/LinkAsset.html | 95 +++++++
.../Asset/Display.html/BeforeProcessArguments | 20 +-
.../Elements/Tabs/Privileged | 5 +
lib/RT/Extension/AWS/Assets.pm | 312 +++++++++++++++------
9 files changed, 393 insertions(+), 90 deletions(-)
create mode 100644 MANIFEST
create mode 100644 html/AWS/LinkAsset.html
hooks/post-receive
--
rt-extension-aws-assets
More information about the Bps-public-commit
mailing list